migrate

2025-12-01 20:14:09 -06:00
parent 3e90af0eb5
commit 6da426af29
73 changed files with 93 additions and 0 deletions
--- a/clusters/cl01tl/helm/argo-workflows/Chart.lock
+++ b/clusters/cl01tl/helm/argo-workflows/Chart.lock
@@ -0,0 +1,12 @@
+dependencies:
+- name: argo-workflows
+  repository: https://argoproj.github.io/argo-helm
+  version: 0.45.28
+- name: argo-events
+  repository: https://argoproj.github.io/argo-helm
+  version: 2.4.17
+- name: postgres-cluster
+  repository: oci://harbor.alexlebens.net/helm-charts
+  version: 6.16.0
+digest: sha256:b00fd479a9d9e606661b3799182c8e24395b4f531f8d2bda87bdc5db16a8d66c
+generated: "2025-12-01T19:55:40.18149-06:00"
--- a/clusters/cl01tl/helm/argo-workflows/Chart.yaml
+++ b/clusters/cl01tl/helm/argo-workflows/Chart.yaml
@@ -0,0 +1,31 @@
+apiVersion: v2
+name: argo-workflows
+version: 1.0.0
+description: Argo Workflows
+keywords:
+  - argo-workflows
+  - argo-events
+  - workflows
+  - events
+home: https://wiki.alexlebens.dev/s/a268508f-d81d-4b4b-8bd5-9058edaea635
+sources:
+  - https://github.com/argoproj/argo-workflows
+  - https://github.com/argoproj/argo-events
+  - https://github.com/cloudnative-pg/cloudnative-pg
+  - https://github.com/argoproj/argo-helm/tree/main/charts
+  - https://gitea.alexlebens.dev/alexlebens/helm-charts/src/branch/main/charts/postgres-cluster
+maintainers:
+  - name: alexlebens
+dependencies:
+  - name: argo-workflows
+    version: 0.45.28
+    repository: https://argoproj.github.io/argo-helm
+  - name: argo-events
+    version: 2.4.17
+    repository: https://argoproj.github.io/argo-helm
+  - name: postgres-cluster
+    alias: postgres-17-cluster
+    version: 6.16.0
+    repository: oci://harbor.alexlebens.net/helm-charts
+icon: https://cdn.jsdelivr.net/gh/selfhst/icons/png/argo-cd.png
+appVersion: v3.6.7
--- a/clusters/cl01tl/helm/argo-workflows/templates/external-secret.yaml
+++ b/clusters/cl01tl/helm/argo-workflows/templates/external-secret.yaml
@@ -0,0 +1,95 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: argo-workflows-oidc-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: argo-workflows-oidc-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: secret
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /authentik/oidc/argo-workflows
+        metadataPolicy: None
+        property: secret
+    - secretKey: client
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /authentik/oidc/argo-workflows
+        metadataPolicy: None
+        property: client
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: argo-workflows-postgresql-17-cluster-backup-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: argo-workflows-postgresql-17-cluster-backup-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: ACCESS_KEY_ID
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /digital-ocean/home-infra/postgres-backups
+        metadataPolicy: None
+        property: access
+    - secretKey: ACCESS_SECRET_KEY
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /digital-ocean/home-infra/postgres-backups
+        metadataPolicy: None
+        property: secret
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: argo-workflows-postgresql-17-cluster-backup-secret-garage
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: argo-workflows-postgresql-17-cluster-backup-secret-garage
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: ACCESS_KEY_ID
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/postgres-backups
+        metadataPolicy: None
+        property: ACCESS_KEY_ID
+    - secretKey: ACCESS_SECRET_KEY
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/postgres-backups
+        metadataPolicy: None
+        property: ACCESS_SECRET_KEY
+    - secretKey: ACCESS_REGION
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/postgres-backups
+        metadataPolicy: None
+        property: ACCESS_REGION
--- a/clusters/cl01tl/helm/argo-workflows/templates/http-route.yaml
+++ b/clusters/cl01tl/helm/argo-workflows/templates/http-route.yaml
@@ -0,0 +1,28 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: http-route-argo-workflows
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: http-route-argo-workflows
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: traefik-gateway
+      namespace: traefik
+  hostnames:
+    - argo-workflows.alexlebens.net
+  rules:
+    - matches:
+      - path:
+          type: PathPrefix
+          value: /
+      backendRefs:
+        - group: ''
+          kind: Service
+          name: argo-workflows-server
+          port: 2746
+          weight: 100
--- a/clusters/cl01tl/helm/argo-workflows/templates/service.yaml
+++ b/clusters/cl01tl/helm/argo-workflows/templates/service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: garage-ps10rp
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: garage-ps10rp
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+  annotations:
+    tailscale.com/tailnet-fqdn: garage-ps10rp.boreal-beaufort.ts.net
+spec:
+  externalName: placeholder
+  type: ExternalName
--- a/clusters/cl01tl/helm/argo-workflows/values.yaml
+++ b/clusters/cl01tl/helm/argo-workflows/values.yaml
@@ -0,0 +1,147 @@
+argo-workflows:
+  controller:
+    metricsConfig:
+      enabled: true
+    persistence:
+      connectionPool:
+        maxIdleConns: 100
+        maxOpenConns: 0
+      nodeStatusOffLoad: true
+      archive: true
+      postgresql:
+        host: argo-workflows-postgresql-17-cluster-rw
+        port: 5432
+        database: app
+        tableName: app
+        userNameSecret:
+          name: argo-workflows-postgresql-17-cluster-app
+          key: username
+        passwordSecret:
+          name: argo-workflows-postgresql-17-cluster-app
+          key: password
+        ssl: false
+        sslMode: disable
+    workflowWorkers: 2
+    workflowTTLWorkers: 1
+    podCleanupWorkers: 1
+    cronWorkflowWorkers: 1
+    resources:
+      requests:
+        cpu: 10m
+        memory: 128Mi
+    serviceMonitor:
+      enabled: true
+    name: workflow-controller
+    workflowNamespaces:
+      - argocd
+      - argo-workflows
+  server:
+    authModes:
+      - sso
+    ingress:
+      enabled: false
+    sso:
+      enabled: true
+      issuer: https://authentik.alexlebens.net/application/o/argo-workflows/
+      clientId:
+        name: argo-workflows-oidc-secret
+        key: client
+      clientSecret:
+        name: argo-workflows-oidc-secret
+        key: secret
+      redirectUrl: https://argo-workflows.alexlebens.net/oauth2/callback
+      rbac:
+        enabled: false
+      scopes:
+        - openid
+        - email
+        - profile
+  useStaticCredentials: true
+  artifactRepository:
+    archiveLogs: false
+    s3: {}
+      # accessKeySecret:
+      #   name: "{{ .Release.Name }}-minio"
+      #   key: accesskey
+      # secretKeySecret:
+      #   name: "{{ .Release.Name }}-minio"
+      #   key: secretkey
+      # insecure: true
+      # bucket:
+      # endpoint:
+      # region:
+      # encryptionOptions:
+      #   enableEncryption: true
+
+argo-events:
+  controller:
+    resources:
+      requests:
+        cpu: 10m
+        memory: 128Mi
+    metrics:
+      enabled: true
+      serviceMonitor:
+        enabled: true
+  webhook:
+    enabled: true
+    resources:
+      requests:
+        cpu: 10m
+        memory: 128Mi
+postgres-17-cluster:
+  mode: recovery
+  cluster:
+    storage:
+      storageClass: local-path
+    walStorage:
+      storageClass: local-path
+    monitoring:
+      enabled: true
+      prometheusRule:
+        enabled: true
+  recovery:
+    method: objectStore
+    objectStore:
+      destinationPath: s3://postgres-backups/cl01tl/argo-workflows/argo-workflows-postgresql-17-cluster
+      endpointURL: http://garage-main.garage:3900
+      index: 1
+      endpointCredentials: argo-workflows-postgresql-17-cluster-backup-secret-garage
+  backup:
+    objectStore:
+      - name: external
+        destinationPath: s3://postgres-backups-ce540ddf106d186bbddca68a/cl01tl/argo-workflows/argo-workflows-postgresql-17-cluster
+        index: 1
+        retentionPolicy: "30d"
+        isWALArchiver: false
+      - name: garage-local
+        destinationPath: s3://postgres-backups/cl01tl/argo-workflows/argo-workflows-postgresql-17-cluster
+        index: 1
+        endpointURL: http://garage-main.garage:3900
+        endpointCredentials: argo-workflows-postgresql-17-cluster-backup-secret-garage
+        endpointCredentialsIncludeRegion: true
+        retentionPolicy: "3d"
+        isWALArchiver: true
+      # - name: garage-remote
+      #   destinationPath: s3://postgres-backups/cl01tl/argo-workflows/argo-workflows-postgresql-17-cluster
+      #   index: 1
+      #   endpointURL: https://garage-ps10rp.boreal-beaufort.ts.net:3900
+      #   endpointCredentials: argo-workflows-postgresql-17-cluster-backup-secret-garage
+      #   endpointCredentialsIncludeRegion: true
+      #   retentionPolicy: "30d"
+      #   data:
+      #     compression: bzip2
+    scheduledBackups:
+      - name: daily-backup
+        suspend: false
+        schedule: "0 0 0 * * *"
+        backupName: external
+      - name: live-backup
+        suspend: false
+        immediate: true
+        schedule: "0 0 0 * * *"
+        backupName: garage-local
+      # - name: weekly-backup
+      #   suspend: true
+      #   schedule: "0 0 4 * * SAT"
+      #   backupName: garage-remote
--- a/clusters/cl01tl/helm/gatus/Chart.lock
+++ b/clusters/cl01tl/helm/gatus/Chart.lock
@@ -0,0 +1,9 @@
+dependencies:
+- name: gatus
+  repository: https://twin.github.io/helm-charts
+  version: 1.4.4
+- name: postgres-cluster
+  repository: oci://harbor.alexlebens.net/helm-charts
+  version: 6.16.0
+digest: sha256:9424c70bc46b5582f19b695196586546c69862ccca8950b906d5532cf03c7792
+generated: "2025-12-01T19:55:44.159963-06:00"
--- a/clusters/cl01tl/helm/gatus/Chart.yaml
+++ b/clusters/cl01tl/helm/gatus/Chart.yaml
@@ -0,0 +1,28 @@
+apiVersion: v2
+name: gatus
+version: 1.0.0
+description: Gatus
+keywords:
+  - gatus
+  - healthcheck
+  - uptime
+  - metrics
+home: https://wiki.alexlebens.dev/s/2a2b0c83-81c7-49e3-aafc-daff4ff23ce2
+sources:
+  - https://github.com/TwiN/gatus
+  - https://github.com/cloudnative-pg/cloudnative-pg
+  - https://github.com/TwiN/gatus/pkgs/container/gatus
+  - https://github.com/TwiN/helm-charts/tree/master/charts/gatus
+  - https://gitea.alexlebens.dev/alexlebens/helm-charts/src/branch/main/charts/postgres-cluster
+maintainers:
+  - name: alexlebens
+dependencies:
+  - name: gatus
+    repository: https://twin.github.io/helm-charts
+    version: 1.4.4
+  - name: postgres-cluster
+    alias: postgres-17-cluster
+    version: 6.16.0
+    repository: oci://harbor.alexlebens.net/helm-charts
+icon: https://cdn.jsdelivr.net/gh/selfhst/icons/png/gatus.png
+appVersion: v5.12.0
--- a/clusters/cl01tl/helm/gatus/templates/external-secret.yaml
+++ b/clusters/cl01tl/helm/gatus/templates/external-secret.yaml
@@ -0,0 +1,118 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: gatus-config-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: gatus-config-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: NTFY_TOKEN
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /ntfy/user/cl01tl
+        metadataPolicy: None
+        property: token
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: gatus-oidc-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: gatus-oidc-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: OIDC_CLIENT_ID
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /authentik/oidc/gatus
+        metadataPolicy: None
+        property: client
+    - secretKey: OIDC_CLIENT_SECRET
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /authentik/oidc/gatus
+        metadataPolicy: None
+        property: secret
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: gatus-postgresql-17-cluster-backup-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: gatus-postgresql-17-cluster-backup-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: ACCESS_KEY_ID
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /digital-ocean/home-infra/postgres-backups
+        metadataPolicy: None
+        property: access
+    - secretKey: ACCESS_SECRET_KEY
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /digital-ocean/home-infra/postgres-backups
+        metadataPolicy: None
+        property: secret
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: gatus-postgresql-17-cluster-backup-secret-garage
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: gatus-postgresql-17-cluster-backup-secret-garage
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: ACCESS_KEY_ID
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/postgres-backups
+        metadataPolicy: None
+        property: ACCESS_KEY_ID
+    - secretKey: ACCESS_SECRET_KEY
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/postgres-backups
+        metadataPolicy: None
+        property: ACCESS_SECRET_KEY
+    - secretKey: ACCESS_REGION
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/postgres-backups
+        metadataPolicy: None
+        property: ACCESS_REGION
--- a/clusters/cl01tl/helm/gatus/templates/http-route.yaml
+++ b/clusters/cl01tl/helm/gatus/templates/http-route.yaml
@@ -0,0 +1,28 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: http-route-gatus
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: http-route-gatus
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: traefik-gateway
+      namespace: traefik
+  hostnames:
+    - gatus.alexlebens.net
+  rules:
+    - matches:
+      - path:
+          type: PathPrefix
+          value: /
+      backendRefs:
+        - group: ''
+          kind: Service
+          name: gatus
+          port: 80
+          weight: 100
--- a/clusters/cl01tl/helm/gatus/values.yaml
+++ b/clusters/cl01tl/helm/gatus/values.yaml
@@ -0,0 +1,423 @@
+gatus:
+  deployment:
+    strategy: Recreate
+  readinessProbe:
+    enabled: true
+  livenessProbe:
+    enabled: true
+  image:
+    repository: ghcr.io/twin/gatus
+    tag: v5.33.0
+  annotations:
+    reloader.stakater.com/auto: "true"
+  service:
+    type: ClusterIP
+    port: 80
+    targetPort: 8080
+    portName: http
+  ingress:
+    enabled: false
+  env:
+    NTFY_TOKEN:
+      valueFrom:
+        secretKeyRef:
+          name: gatus-config-secret
+          key: NTFY_TOKEN
+    OIDC_CLIENT_ID:
+      valueFrom:
+        secretKeyRef:
+          name: gatus-oidc-secret
+          key: OIDC_CLIENT_ID
+    OIDC_CLIENT_SECRET:
+      valueFrom:
+        secretKeyRef:
+          name: gatus-oidc-secret
+          key: OIDC_CLIENT_SECRET
+    POSTGRES_USER:
+      valueFrom:
+        secretKeyRef:
+          name: gatus-postgresql-17-cluster-app
+          key: username
+    POSTGRES_PASSWORD:
+      valueFrom:
+        secretKeyRef:
+          name: gatus-postgresql-17-cluster-app
+          key: password
+    POSTGRES_HOST:
+      valueFrom:
+        secretKeyRef:
+          name: gatus-postgresql-17-cluster-app
+          key: host
+    POSTGRES_PORT:
+      valueFrom:
+        secretKeyRef:
+          name: gatus-postgresql-17-cluster-app
+          key: port
+    POSTGRES_DB:
+      valueFrom:
+        secretKeyRef:
+          name: gatus-postgresql-17-cluster-app
+          key: dbname
+  resources:
+    requests:
+      cpu: 10m
+      memory: 128Mi
+  persistence:
+    enabled: true
+    size: 1Gi
+    mountPath: /data
+    accessModes:
+      - ReadWriteOnce
+    finalizers:
+      - kubernetes.io/pvc-protection
+    storageClassName: ceph-block
+  serviceMonitor:
+    enabled: true
+    interval: 1m
+    path: /metrics
+    scheme: http
+    scrapeTimeout: 30s
+  networkPolicy:
+    enabled: false
+  config:
+    metrics: true
+    connectivity:
+      checker:
+        target: 1.1.1.1:53
+        interval: 60s
+    alerting:
+      ntfy:
+        topic: "gatus-alerts"
+        priority: 3
+        url: http://ntfy.ntfy
+        token: ${NTFY_TOKEN}
+        default-alert:
+          failure-threshold: 5
+          send-on-resolved: true
+        click: "https://gatus.alexlebens.net"
+    security:
+      oidc:
+        issuer-url: https://authentik.alexlebens.net/application/o/gatus/
+        client-id: ${OIDC_CLIENT_ID}
+        client-secret: ${OIDC_CLIENT_SECRET}
+        redirect-url: https://gatus.alexlebens.net/authorization-code/callback
+        scopes: [openid]
+    storage:
+      type: postgres
+      path: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}?sslmode=disable"
+    default-endpoint: &defaults
+      group: core
+      conditions:
+        - "[STATUS] == 200"
+        - "[CERTIFICATE_EXPIRATION] > 240h"
+      interval: 30s
+      alerts:
+        - type: ntfy
+    endpoints:
+      - name: plex
+        url: http://plex.alexlebens.net
+        <<: *defaults
+        conditions:
+          - "[STATUS] == 401"
+      - name: jellyfin
+        url: https://jellyfin.alexlebens.net
+        <<: *defaults
+      - name: overseerr
+        url: https://overseerr.alexlebens.net
+        <<: *defaults
+      - name: yamtrack
+        url: https://yamtrack.alexlebens.net
+        <<: *defaults
+      - name: tubearchivist
+        url: https://tubearchivist.alexlebens.net
+        <<: *defaults
+      - name: immich
+        url: https://immich.alexlebens.net
+        <<: *defaults
+      - name: photoview
+        url: https://photoview.alexlebens.net
+        <<: *defaults
+      - name: audiobookshelf
+        url: https://audiobookshelf.alexlebens.net
+        <<: *defaults
+      - name: home-assistant
+        url: https://home-assistant.alexlebens.net
+        <<: *defaults
+      - name: actual
+        url: https://actual.alexlebens.net
+        <<: *defaults
+      - name: ollama
+        url: https://ollama.alexlebens.net
+        <<: *defaults
+      - name: searxng
+        url: https://searxng.alexlebens.net
+        <<: *defaults
+      - name: roundcube
+        url: https://mail.alexlebens.net
+        <<: *defaults
+      - name: kiwix
+        url: https://kiwix.alexlebens.net
+        <<: *defaults
+      - name: gitea
+        url: https://gitea.alexlebens.net
+        <<: *defaults
+      - name: home-assistant-code-server
+        url: https://home-assistant-code-server.alexlebens.net
+        <<: *defaults
+      - name: argocd
+        url: https://argocd.alexlebens.net
+        <<: *defaults
+      - name: komodo
+        url: https://komodo.alexlebens.net
+        <<: *defaults
+      - name: argo-workflows
+        url: https://argo-workflows.alexlebens.net
+        <<: *defaults
+      - name: n8n
+        url: https://n8n.alexlebens.net
+        <<: *defaults
+      - name: omni-tools
+        url: https://omni-tools.alexlebens.net
+        <<: *defaults
+      - name: headlamp
+        url: https://headlamp.alexlebens.net
+        <<: *defaults
+      - name: hubble
+        url: https://hubble.alexlebens.net
+        <<: *defaults
+      - name: grafana
+        url: https://grafana.alexlebens.net
+        <<: *defaults
+      - name: prometheus
+        url: https://prometheus.alexlebens.net
+        <<: *defaults
+      - name: alertmanager
+        url: https://alertmanager.alexlebens.net
+        <<: *defaults
+      - name: tautulli
+        url: https://tautulli.alexlebens.net
+        <<: *defaults
+      - name: jellystat
+        url: https://jellystat.alexlebens.net
+        <<: *defaults
+      - name: authentik
+        url: https://authentik.alexlebens.net
+        <<: *defaults
+      - name: stalwart
+        url: https://stalwart.alexlebens.net
+        <<: *defaults
+      - name: ntfy
+        url: https://ntfy.alexlebens.net
+        <<: *defaults
+      - name: traefik-cl01tl
+        url: https://traefik-cl01tl.alexlebens.net/dashboard/#/
+        <<: *defaults
+      - name: harbor
+        url: https://harbor.alexlebens.net
+        <<: *defaults
+      - name: unifi
+        url: https://unifi.alexlebens.net
+        <<: *defaults
+      - name: synology
+        url: https://synology.alexlebens.net
+        <<: *defaults
+        client:
+          insecure: true
+        conditions:
+          - "[CONNECTED] == true"
+      - name: hdhr
+        url: http://hdhr.alexlebens.net
+        <<: *defaults
+        conditions:
+          - "[STATUS] == 200"
+      - name: pikvm
+        url: https://pikvm.alexlebens.net/login/
+        <<: *defaults
+        client:
+          insecure: true
+        conditions:
+          - "[CONNECTED] == true"
+      - name: shelly
+        url: http://it05sp.alexlebens.net
+        <<: *defaults
+        conditions:
+          - "[STATUS] == 200"
+      - name: ceph
+        url: https://ceph.alexlebens.net
+        <<: *defaults
+      - name: pgadmin
+        url: https://pgadmin.alexlebens.net
+        <<: *defaults
+      - name: whodb
+        url: https://whodb.alexlebens.net
+        <<: *defaults
+      - name: vault
+        url: https://vault.alexlebens.net
+        <<: *defaults
+      - name: sonarr
+        url: https://sonarr.alexlebens.net
+        <<: *defaults
+      - name: sonarr-4k
+        url: https://sonarr-4k.alexlebens.net
+        <<: *defaults
+      - name: sonarr-anime
+        url: https://sonarr-anime.alexlebens.net
+        <<: *defaults
+      - name: radarr
+        url: https://radarr.alexlebens.net
+        <<: *defaults
+      - name: radarr-4k
+        url: https://radarr-4k.alexlebens.net
+        <<: *defaults
+      - name: radarr-anime
+        url: https://radarr-anime.alexlebens.net
+        <<: *defaults
+      - name: radarr-standup
+        url: https://radarr-standup.alexlebens.net
+        <<: *defaults
+      - name: lidarr
+        url: https://lidarr.alexlebens.net
+        <<: *defaults
+      - name: lidatube
+        url: https://lidatube.alexlebens.net
+        <<: *defaults
+      - name: slskd
+        url: https://slskd.alexlebens.net
+        <<: *defaults
+      - name: qui
+        url: https://qui.alexlebens.net
+        <<: *defaults
+      - name: qbittorrent
+        url: https://qbittorrent.alexlebens.net
+        <<: *defaults
+      - name: prowlarr
+        url: https://prowlarr.alexlebens.net
+        <<: *defaults
+      - name: bazarr
+        url: https://bazarr.alexlebens.net
+        <<: *defaults
+        conditions:
+          - "[STATUS] == 401"
+      - name: huntarr
+        url: https://huntarr.alexlebens.net
+        <<: *defaults
+      - name: tdarr
+        url: https://tdarr.alexlebens.net
+        <<: *defaults
+      - name: www
+        url: https://www.alexlebens.dev
+        <<: *defaults
+        group: external
+      - name: directus
+        url: https://directus.alexlebens.dev
+        <<: *defaults
+        group: external
+      - name: postiz
+        url: https://postiz.alexlebens.dev
+        <<: *defaults
+        interval: 120s
+        group: external
+      - name: matrix
+        url: https://chat.alexlebens.dev
+        <<: *defaults
+        group: external
+      - name: outline
+        url: https://wiki.alexlebens.dev
+        <<: *defaults
+        group: external
+      - name: vaultwarden
+        url: https://passwords.alexlebens.dev
+        <<: *defaults
+        group: external
+      - name: karakeep
+        url: https://karakeep.alexlebens.dev
+        <<: *defaults
+        group: external
+      - name: freshrss
+        url: https://rss.alexlebens.dev/i/
+        <<: *defaults
+        group: external
+        conditions:
+          - "[STATUS] == 401"
+      - name: gitea-external
+        url: https://gitea.alexlebens.dev
+        <<: *defaults
+        group: external
+      - name: codeserver
+        url: https://codeserver.alexlebens.dev
+        <<: *defaults
+        group: external
+      - name: public homepage
+        url: https://home.alexlebens.dev
+        <<: *defaults
+        group: external
+      - name: discord
+        group: public
+        url: https://discord.com/app
+        conditions:
+          - "[STATUS] == 200"
+          - "[RESPONSE_TIME] < 400"
+        interval: 10s
+      - name: reddit
+        group: public
+        url: https://reddit.com
+        conditions:
+          - "[STATUS] == 200"
+          - "[RESPONSE_TIME] < 400"
+        interval: 10s
+postgres-17-cluster:
+  mode: recovery
+  cluster:
+    storage:
+      storageClass: local-path
+    walStorage:
+      storageClass: local-path
+    monitoring:
+      enabled: true
+      prometheusRule:
+        enabled: true
+  recovery:
+    method: objectStore
+    objectStore:
+      destinationPath: s3://postgres-backups/cl01tl/gatus/gatus-postgresql-17-cluster
+      endpointURL: http://garage-main.garage:3900
+      index: 1
+      endpointCredentials: gatus-postgresql-17-cluster-backup-secret-garage
+  backup:
+    objectStore:
+      - name: external
+        destinationPath: s3://postgres-backups-ce540ddf106d186bbddca68a/cl01tl/gatus/gatus-postgresql-17-cluster
+        index: 2
+        retentionPolicy: "30d"
+        isWALArchiver: false
+      - name: garage-local
+        destinationPath: s3://postgres-backups/cl01tl/gatus/gatus-postgresql-17-cluster
+        index: 1
+        endpointURL: http://garage-main.garage:3900
+        endpointCredentials: gatus-postgresql-17-cluster-backup-secret-garage
+        endpointCredentialsIncludeRegion: true
+        retentionPolicy: "3d"
+        isWALArchiver: true
+      # - name: garage-remote
+      #   destinationPath: s3://postgres-backups/cl01tl/gatus/gatus-postgresql-17-cluster
+      #   index: 1
+      #   endpointURL: https://garage-ps10rp.boreal-beaufort.ts.net:3900
+      #   endpointCredentials: gatus-postgresql-17-cluster-backup-secret-garage
+      #   retentionPolicy: "30d"
+      #   data:
+      #     compression: bzip2
+      #     jobs: 2
+    scheduledBackups:
+      - name: daily-backup
+        suspend: false
+        schedule: "0 0 0 * * *"
+        backupName: external
+      - name: live-backup
+        suspend: false
+        immediate: true
+        schedule: "0 0 0 * * *"
+        backupName: garage-local
+      # - name: weekly-backup
+      #   suspend: false
+      #   schedule: "0 0 4 * * SAT"
+      #   backupName: garage-remote
--- a/clusters/cl01tl/helm/grafana-operator/Chart.lock
+++ b/clusters/cl01tl/helm/grafana-operator/Chart.lock
@@ -0,0 +1,9 @@
+dependencies:
+- name: grafana-operator
+  repository: https://grafana.github.io/helm-charts
+  version: v5.20.0
+- name: postgres-cluster
+  repository: oci://harbor.alexlebens.net/helm-charts
+  version: 6.16.0
+digest: sha256:d70b284b771af610116564e5f47d72415ab0d3520ecb552159ee8dc8acd2ab46
+generated: "2025-12-01T19:55:46.485062-06:00"
--- a/clusters/cl01tl/helm/grafana-operator/Chart.yaml
+++ b/clusters/cl01tl/helm/grafana-operator/Chart.yaml
@@ -0,0 +1,27 @@
+apiVersion: v2
+name: grafana-operator
+version: 1.0.0
+description: Grafana Operator
+keywords:
+  - grafana-operator
+  - dashboard
+  - metrics
+  - logs
+home: https://wiki.alexlebens.dev/s/3e5723e1-2ab7-45ab-b496-b8854907fa39
+sources:
+  - https://github.com/grafana/grafana-operator
+  - https://github.com/cloudnative-pg/cloudnative-pg
+  - https://github.com/grafana/grafana-operator/tree/master/deploy/helm/grafana-operator
+  - https://gitea.alexlebens.dev/alexlebens/helm-charts/src/branch/main/charts/postgres-cluster
+maintainers:
+  - name: alexlebens
+dependencies:
+  - name: grafana-operator
+    version: v5.20.0
+    repository: https://grafana.github.io/helm-charts
+  - name: postgres-cluster
+    alias: postgres-17-cluster
+    version: 6.16.0
+    repository: oci://harbor.alexlebens.net/helm-charts
+icon: https://cdn.jsdelivr.net/gh/selfhst/icons/png/grafana.png
+appVersion: v5.18.0
--- a/clusters/cl01tl/helm/grafana-operator/templates/external-secret.yaml
+++ b/clusters/cl01tl/helm/grafana-operator/templates/external-secret.yaml
@@ -0,0 +1,125 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: grafana-auth-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-auth-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: admin-user
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /cl01tl/grafana/auth
+        metadataPolicy: None
+        property: admin-user
+    - secretKey: admin-password
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /cl01tl/grafana/auth
+        metadataPolicy: None
+        property: admin-password
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: grafana-oauth-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-oauth-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: AUTH_CLIENT_ID
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /authentik/oidc/grafana
+        metadataPolicy: None
+        property: client
+    - secretKey: AUTH_CLIENT_SECRET
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /authentik/oidc/grafana
+        metadataPolicy: None
+        property: secret
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: grafana-operator-postgresql-17-cluster-backup-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-operator-postgresql-17-cluster-backup-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: ACCESS_KEY_ID
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /digital-ocean/home-infra/postgres-backups
+        metadataPolicy: None
+        property: access
+    - secretKey: ACCESS_SECRET_KEY
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /digital-ocean/home-infra/postgres-backups
+        metadataPolicy: None
+        property: secret
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: grafana-operator-postgresql-17-cluster-backup-secret-garage
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-operator-postgresql-17-cluster-backup-secret-garage
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: ACCESS_KEY_ID
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/postgres-backups
+        metadataPolicy: None
+        property: ACCESS_KEY_ID
+    - secretKey: ACCESS_SECRET_KEY
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/postgres-backups
+        metadataPolicy: None
+        property: ACCESS_SECRET_KEY
+    - secretKey: ACCESS_REGION
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/postgres-backups
+        metadataPolicy: None
+        property: ACCESS_REGION
--- a/clusters/cl01tl/helm/grafana-operator/templates/grafana-dashboard.yaml
+++ b/clusters/cl01tl/helm/grafana-operator/templates/grafana-dashboard.yaml
@@ -0,0 +1,644 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-ceph
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-ceph
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-system
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/system/ceph.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-coredns
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-coredns
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-system
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/system/coredns.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-etcd
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-etcd
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-system
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/system/etcd.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-garage
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-garage
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-system
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/system/garage.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-loki
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-loki
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-system
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/system/loki.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-node-full
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-node-full
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-system
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/system/node-full.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-node-short
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-node-short
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-system
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/system/node-short.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-argocd
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-argocd
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-service
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/service/argocd.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-blocky
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-blocky
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-service
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/service/blocky.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-cert-manager
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-cert-manager
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-service
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/service/cert-manager.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-cloudnative-pg
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-cloudnative-pg
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-service
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/service/cloudnative-pg.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-descheduler
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-descheduler
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-service
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/service/descheduler.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-gatus
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-gatus
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-service
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/service/gatus.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-operator
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-operator
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-service
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/service/grafana-operator.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-harbor
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-harbor
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-service
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/service/harbor.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-redis-replication
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-redis-replication
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-service
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/service/redis-replication.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-redis-operator
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-redis-operator
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-service
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/service/redis-operator.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-speedtest-exporter
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-speedtest-exporter
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-service
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/service/speedtest-exporter.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-spegel
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-spegel
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-service
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/service/spegel.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-traefik
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-traefik
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-service
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/service/traefik.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-trivy
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-trivy
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-service
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/service/trivy.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-unpoller
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-unpoller
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-service
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/service/unpoller.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-volsync
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-volsync
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-service
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/service/volsync.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-s3
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-s3
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-platform
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/platform/s3.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-authentik
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-authentik
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-platform
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/platform/authentik.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-gitea
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-gitea
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-platform
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/platform/gitea.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-ntfy
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-ntfy
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-platform
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/platform/ntfy.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-qbittorrent
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-qbittorrent
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-platform
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/platform/qbittorrent.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-vault
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-vault
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-platform
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/platform/vault.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-airgradient
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-airgradient
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-iot
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/iot/airgradient.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-server-power-consumption
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-server-power-consumption
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-iot
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/iot/server-power-consumption.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-immich
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-immich
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-application
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/application/immich.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-radarr
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-radarr
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-application
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/application/radarr.json
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-sonarr
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-dashboard-sonarr
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  contentCacheDuration: 1h
+  folderUID: grafana-folder-application
+  resyncPeriod: 1h
+  url: http://gitea-http.gitea:3000/alexlebens/grafana-dashboards/raw/branch/main/dashboards/application/sonarr.json
--- a/clusters/cl01tl/helm/grafana-operator/templates/grafana-datasource.yaml
+++ b/clusters/cl01tl/helm/grafana-operator/templates/grafana-datasource.yaml
@@ -0,0 +1,48 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDatasource
+metadata:
+  name: grafana-datasource-prometheus
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-datasource-prometheus
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  datasource:
+    name: Prometheus
+    type: prometheus
+    url: http://kube-prometheus-stack-prometheus.kube-prometheus-stack:9090/
+    access: proxy
+    isDefault: true
+    jsonData:
+      timeInterval: 30s
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  plugins:
+    - name: camptocamp-prometheus-alertmanager-datasource
+      version: 2.1.0
+  resyncPeriod: 30s
+  uid: kube-prometheus-stack
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDatasource
+metadata:
+  name: grafana-datasource-loki
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-datasource-loki
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  datasource:
+    name: Loki
+    type: loki
+    url: http://loki.loki:3100/
+    access: proxy
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  resyncPeriod: 30s
+  uid: loki
--- a/clusters/cl01tl/helm/grafana-operator/templates/grafana-folder.yaml
+++ b/clusters/cl01tl/helm/grafana-operator/templates/grafana-folder.yaml
@@ -0,0 +1,173 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaFolder
+metadata:
+  name: grafana-folder-application
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-folder-application
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  title: Application
+  uid: grafana-folder-application
+  resyncPeriod: 30s
+  permissions: |
+    {
+      "items": [
+        {
+          "role": "Admin",
+          "permission": 4
+        },
+        {
+          "role": "Editor",
+          "permission": 2
+        },
+        {
+          "role": "Viewer",
+          "permission": 1
+        }
+      ]
+    }
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaFolder
+metadata:
+  name: grafana-folder-iot
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-folder-iot
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  title: IoT
+  uid: grafana-folder-iot
+  resyncPeriod: 30s
+  permissions: |
+    {
+      "items": [
+        {
+          "role": "Admin",
+          "permission": 4
+        },
+        {
+          "role": "Editor",
+          "permission": 2
+        },
+        {
+          "role": "Viewer",
+          "permission": 1
+        }
+      ]
+    }
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaFolder
+metadata:
+  name: grafana-folder-platform
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-folder-platform
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  title: Platform
+  uid: grafana-folder-platform
+  resyncPeriod: 30s
+  permissions: |
+    {
+      "items": [
+        {
+          "role": "Admin",
+          "permission": 4
+        },
+        {
+          "role": "Editor",
+          "permission": 2
+        },
+        {
+          "role": "Viewer",
+          "permission": 1
+        }
+      ]
+    }
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaFolder
+metadata:
+  name: grafana-folder-service
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-folder-service
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app: grafana-main
+  title: Service
+  uid: grafana-folder-service
+  resyncPeriod: 30s
+  permissions: |
+    {
+      "items": [
+        {
+          "role": "Admin",
+          "permission": 4
+        },
+        {
+          "role": "Editor",
+          "permission": 2
+        },
+        {
+          "role": "Viewer",
+          "permission": 1
+        }
+      ]
+    }
+
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaFolder
+metadata:
+  name: grafana-folder-system
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-folder-system
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  instanceSelector:
+    matchLabels:
+      app.kubernetes.io/name: grafana-main
+  title: System
+  uid: grafana-folder-system
+  resyncPeriod: 30s
+  permissions: |
+    {
+      "items": [
+        {
+          "role": "Admin",
+          "permission": 4
+        },
+        {
+          "role": "Editor",
+          "permission": 2
+        },
+        {
+          "role": "Viewer",
+          "permission": 1
+        }
+      ]
+    }
--- a/clusters/cl01tl/helm/grafana-operator/templates/grafana.yaml
+++ b/clusters/cl01tl/helm/grafana-operator/templates/grafana.yaml
@@ -0,0 +1,109 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: Grafana
+metadata:
+  name: grafana-main
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: grafana-main
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+    app: grafana-main
+spec:
+  config:
+    analytics:
+      enabled: "false"
+      check_for_updates: "false"
+      reporting_enabled: "false"
+    server:
+      domain: alexlebens.net
+      root_url: https://grafana.alexlebens.net
+    log:
+      mode: "console"
+    users:
+      auto_assign_org: "true"
+      auto_assign_org_id: "1"
+    auth:
+      disable_login_form: "true"
+      auto_login: "true"
+      signout_redirect_url: https://authentik.alexlebens.net/application/o/grafana/end-session/
+    auth.generic_oauth:
+      enabled: "true"
+      name: Authentik
+      allow_sign_up: "true"
+      client_id: ${AUTH_CLIENT_ID}
+      client_secret: ${AUTH_CLIENT_SECRET}
+      scopes: openid profile email
+      auth_url: https://authentik.alexlebens.net/application/o/authorize/
+      token_url: https://authentik.alexlebens.net/application/o/token/
+      api_url: https://authentik.alexlebens.net/application/o/userinfo/
+      role_attribute_path: contains(groups, 'Grafana Admins') && 'Admin' || contains(groups, 'Grafana Editors') && 'Editor' || 'Viewer'
+    database:
+      type: postgres
+      host: "${DB_HOST}:${DB_PORT}"
+      name: ${DB_DATABASE}
+      user: ${DB_USER}
+      password: ${DB_PASSWORD}
+    remote_cache:
+      type: redis
+      connstr: addr=redis-replication-remote-cache-master.grafana-operator:6379,pool_size=100,db=0,ssl=false
+    unified_alerting:
+      enabled: "true"
+      ha_redis_address: redis-replication-unified-alerting-master.grafana-operator:6379
+  deployment:
+    spec:
+      replicas: 3
+      template:
+        spec:
+          containers:
+            - name: grafana
+              image: grafana/grafana:12.0.0
+              resources:
+                requests:
+                  cpu: 100m
+                  memory: 128Mi
+              env:
+                - name: AUTH_CLIENT_ID
+                  valueFrom:
+                    secretKeyRef:
+                      name: grafana-oauth-secret
+                      key: AUTH_CLIENT_ID
+                - name: AUTH_CLIENT_SECRET
+                  valueFrom:
+                    secretKeyRef:
+                      name: grafana-oauth-secret
+                      key: AUTH_CLIENT_SECRET
+                - name: ADMIN_USER
+                  valueFrom:
+                    secretKeyRef:
+                      name: grafana-auth-secret
+                      key: admin-user
+                - name: ADMIN_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: grafana-auth-secret
+                      key: admin-password
+                - name: DB_HOST
+                  valueFrom:
+                    secretKeyRef:
+                      name: grafana-operator-postgresql-17-cluster-app
+                      key: host
+                - name: DB_DATABASE
+                  valueFrom:
+                    secretKeyRef:
+                      name: grafana-operator-postgresql-17-cluster-app
+                      key: dbname
+                - name: DB_PORT
+                  valueFrom:
+                    secretKeyRef:
+                      name: grafana-operator-postgresql-17-cluster-app
+                      key: port
+                - name: DB_USER
+                  valueFrom:
+                    secretKeyRef:
+                      name: grafana-operator-postgresql-17-cluster-app
+                      key: user
+                - name: DB_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: grafana-operator-postgresql-17-cluster-app
+                      key: password
--- a/clusters/cl01tl/helm/grafana-operator/templates/http-route.yaml
+++ b/clusters/cl01tl/helm/grafana-operator/templates/http-route.yaml
@@ -0,0 +1,28 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: http-route-grafana
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: http-route-grafana
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: traefik-gateway
+      namespace: traefik
+  hostnames:
+    - grafana.alexlebens.net
+  rules:
+    - matches:
+      - path:
+          type: PathPrefix
+          value: /
+      backendRefs:
+        - group: ''
+          kind: Service
+          name: grafana-main-service
+          port: 3000
+          weight: 100
--- a/clusters/cl01tl/helm/grafana-operator/templates/redis-replication.yaml
+++ b/clusters/cl01tl/helm/grafana-operator/templates/redis-replication.yaml
@@ -0,0 +1,66 @@
+apiVersion: redis.redis.opstreelabs.in/v1beta2
+kind: RedisReplication
+metadata:
+  name: redis-replication-unified-alerting
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: redis-replication-unified-alerting
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  clusterSize: 3
+  podSecurityContext:
+    runAsUser: 1000
+    fsGroup: 1000
+  kubernetesConfig:
+    image: quay.io/opstree/redis:v8.0.3
+    imagePullPolicy: IfNotPresent
+    resources:
+      requests:
+        cpu: 50m
+        memory: 128Mi
+  storage:
+    volumeClaimTemplate:
+      spec:
+        storageClassName: ceph-block
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 1Gi
+  redisExporter:
+    enabled: true
+    image: quay.io/opstree/redis-exporter:v1.48.0
+
+---
+apiVersion: redis.redis.opstreelabs.in/v1beta2
+kind: RedisReplication
+metadata:
+  name: redis-replication-remote-cache
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: redis-replication-remote-cache
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  clusterSize: 3
+  podSecurityContext:
+    runAsUser: 1000
+    fsGroup: 1000
+  kubernetesConfig:
+    image: quay.io/opstree/redis:v8.0.3
+    imagePullPolicy: IfNotPresent
+    resources:
+      requests:
+        cpu: 50m
+        memory: 128Mi
+  storage:
+    volumeClaimTemplate:
+      spec:
+        storageClassName: ceph-block
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 1Gi
+  redisExporter:
+    enabled: true
+    image: quay.io/opstree/redis-exporter:v1.48.0
--- a/clusters/cl01tl/helm/grafana-operator/templates/service-monitor.yaml
+++ b/clusters/cl01tl/helm/grafana-operator/templates/service-monitor.yaml
@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: redis-replication-grafana-operator
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: redis-replication-grafana-operator
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+    redis-operator: "true"
+    env: production
+spec:
+  selector:
+    matchLabels:
+      redis_setup_type: replication
+  endpoints:
+    - port: redis-exporter
+      interval: 30s
+      scrapeTimeout: 10s
--- a/clusters/cl01tl/helm/grafana-operator/values.yaml
+++ b/clusters/cl01tl/helm/grafana-operator/values.yaml
@@ -0,0 +1,70 @@
+grafana-operator:
+  replicas: 2
+  serviceAccount:
+    create: true
+  rbac:
+    create: true
+  resources:
+    requests:
+      cpu: 10m
+      memory: 64Mi
+  serviceMonitor:
+    enabled: true
+  dashboard:
+    enabled: false
+postgres-17-cluster:
+  mode: recovery
+  cluster:
+    storage:
+      storageClass: local-path
+    walStorage:
+      storageClass: local-path
+    monitoring:
+      enabled: true
+      prometheusRule:
+        enabled: true
+  recovery:
+    method: objectStore
+    objectStore:
+      destinationPath: s3://postgres-backups/cl01tl/grafana-operator/grafana-operator-postgresql-17-cluster
+      endpointURL: http://garage-main.garage:3900
+      index: 1
+      endpointCredentials: grafana-operator-postgresql-17-cluster-backup-secret-garage
+  backup:
+    objectStore:
+      - name: external
+        destinationPath: s3://postgres-backups-ce540ddf106d186bbddca68a/cl01tl/grafana-operator/grafana-operator-postgresql-17-cluster
+        index: 2
+        retentionPolicy: "30d"
+        isWALArchiver: false
+      - name: garage-local
+        destinationPath: s3://postgres-backups/cl01tl/grafana-operator/grafana-operator-postgresql-17-cluster
+        index: 1
+        endpointURL: http://garage-main.garage:3900
+        endpointCredentials: grafana-operator-postgresql-17-cluster-backup-secret-garage
+        endpointCredentialsIncludeRegion: true
+        retentionPolicy: "3d"
+        isWALArchiver: true
+      # - name: garage-remote
+      #   destinationPath: s3://postgres-backups/cl01tl/grafana-operator/grafana-operator-postgresql-17-cluster
+      #   index: 1
+      #   endpointURL: https://garage-ps10rp.boreal-beaufort.ts.net:3900
+      #   endpointCredentials: grafana-operator-postgresql-17-cluster-backup-secret-garage
+      #   retentionPolicy: "30d"
+      #   data:
+      #     compression: bzip2
+      #     jobs: 2
+    scheduledBackups:
+      - name: daily-backup
+        suspend: false
+        schedule: "0 0 0 * * *"
+        backupName: external
+      - name: live-backup
+        suspend: false
+        immediate: true
+        schedule: "0 0 0 * * *"
+        backupName: garage-local
+      # - name: weekly-backup
+      #   suspend: false
+      #   schedule: "0 0 4 * * SAT"
+      #   backupName: garage-remote
--- a/clusters/cl01tl/helm/headlamp/Chart.lock
+++ b/clusters/cl01tl/helm/headlamp/Chart.lock
@@ -0,0 +1,6 @@
+dependencies:
+- name: headlamp
+  repository: https://kubernetes-sigs.github.io/headlamp/
+  version: 0.38.0
+digest: sha256:3f4c6bb308a1e5e757368ea9eee902d5ade7d33881c0f6c8402d6ed41641e260
+generated: "2025-12-01T19:55:48.64361-06:00"
--- a/clusters/cl01tl/helm/headlamp/Chart.yaml
+++ b/clusters/cl01tl/helm/headlamp/Chart.yaml
@@ -0,0 +1,20 @@
+apiVersion: v2
+name: headlamp
+version: 1.0.0
+description: Headlamp
+keywords:
+  - headlamp
+  - dashboard
+  - kubernetes
+home: https://wiki.alexlebens.dev/s/6cc43960-78df-459d-aab6-433844249243
+sources:
+  - https://github.com/headlamp-k8s/headlamp
+  - https://github.com/headlamp-k8s/headlamp/tree/main/charts/headlamp
+maintainers:
+  - name: alexlebens
+dependencies:
+  - name: headlamp
+    version: 0.38.0
+    repository: https://kubernetes-sigs.github.io/headlamp/
+icon: https://cdn.jsdelivr.net/gh/selfhst/icons/png/headlamp.png
+appVersion: 0.37.0
--- a/clusters/cl01tl/helm/headlamp/templates/cluster-role-binding.yaml
+++ b/clusters/cl01tl/helm/headlamp/templates/cluster-role-binding.yaml
@@ -0,0 +1,20 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cluster-admin-oidc
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: cluster-admin-oidc
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: User
+    name: alexanderlebens@gmail.com
+    apiGroup: rbac.authorization.k8s.io
+  - kind: ServiceAccount
+    name: headlamp-admin
+    namespace: headlamp
--- a/clusters/cl01tl/helm/headlamp/templates/external-secret.yaml
+++ b/clusters/cl01tl/helm/headlamp/templates/external-secret.yaml
@@ -0,0 +1,56 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: headlamp-oidc-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: headlamp-oidc-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: OIDC_CLIENT_ID
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /authentik/oidc/headlamp
+        metadataPolicy: None
+        property: client
+    - secretKey: OIDC_CLIENT_SECRET
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /authentik/oidc/headlamp
+        metadataPolicy: None
+        property: secret
+    - secretKey: OIDC_ISSUER_URL
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /authentik/oidc/headlamp
+        metadataPolicy: None
+        property: issuer
+    - secretKey: OIDC_SCOPES
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /authentik/oidc/headlamp
+        metadataPolicy: None
+        property: scopes
+    - secretKey: OIDC_VALIDATOR_ISSUER_URL
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /authentik/oidc/headlamp
+        metadataPolicy: None
+        property: validator-issuer-url
+    - secretKey: OIDC_VALIDATOR_CLIENT_ID
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /authentik/oidc/headlamp
+        metadataPolicy: None
+        property: validator-client-id
--- a/clusters/cl01tl/helm/headlamp/templates/http-route.yaml
+++ b/clusters/cl01tl/helm/headlamp/templates/http-route.yaml
@@ -0,0 +1,28 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: https-route-headlamp
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: https-route-headlamp
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: traefik-gateway
+      namespace: traefik
+  hostnames:
+    - headlamp.alexlebens.net
+  rules:
+    - matches:
+      - path:
+          type: PathPrefix
+          value: /
+      backendRefs:
+        - group: ''
+          kind: Service
+          name: headlamp
+          port: 80
+          weight: 100
--- a/clusters/cl01tl/helm/headlamp/templates/service-account.yaml
+++ b/clusters/cl01tl/helm/headlamp/templates/service-account.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: headlamp-admin
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: headlamp-admin
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
--- a/clusters/cl01tl/helm/headlamp/values.yaml
+++ b/clusters/cl01tl/helm/headlamp/values.yaml
@@ -0,0 +1,31 @@
+headlamp:
+  replicaCount: 2
+  config:
+    oidc:
+      secret:
+        create: false
+      externalSecret:
+        enabled: true
+        name: headlamp-oidc-secret
+    watchPlugins: true
+  resources:
+    requests:
+      cpu: 10m
+      memory: 128Mi
+  pluginsManager:
+    enabled: true
+    securityContext:
+      readOnlyRootFilesystem: false
+      runAsNonRoot: false
+      runAsUser: 0
+    configContent: |
+      plugins:
+        - name: cert-manager
+          source: https://artifacthub.io/packages/headlamp/headlamp-plugins/headlamp_cert-manager
+          version: 0.1.0
+        - name: trivy
+          source: https://artifacthub.io/packages/headlamp/headlamp-trivy/headlamp_trivy
+          version: 0.3.1
+      installOptions:
+        parallel: true
+        maxConcurrent: 2
--- a/clusters/cl01tl/helm/komodo/Chart.lock
+++ b/clusters/cl01tl/helm/komodo/Chart.lock
@@ -0,0 +1,9 @@
+dependencies:
+- name: app-template
+  repository: https://bjw-s-labs.github.io/helm-charts/
+  version: 4.4.0
+- name: postgres-cluster
+  repository: oci://harbor.alexlebens.net/helm-charts
+  version: 6.16.0
+digest: sha256:2e3d60887a3f063401c22e5403ed9d55eed07ffb7fbe4bac39eaafc951830f18
+generated: "2025-12-01T19:55:50.498761-06:00"
--- a/clusters/cl01tl/helm/komodo/Chart.yaml
+++ b/clusters/cl01tl/helm/komodo/Chart.yaml
@@ -0,0 +1,29 @@
+apiVersion: v2
+name: komodo
+version: 1.0.0
+description: Komodo
+keywords:
+  - komodo
+  - deployment
+  - dashboard
+  - docker-compose
+home: https://wiki.alexlebens.dev/s/bb7eb683-b5c7-4f50-9f2c-e8e57dc67c81
+sources:
+  - https://github.com/mbecker20/komodo
+  - https://github.com/cloudnative-pg/cloudnative-pg
+  - https://github.com/moghtech/komodo/pkgs/container/komodo-core
+  - https://github.com/bjw-s-labs/helm-charts/tree/main/charts/other/app-template
+  - https://gitea.alexlebens.dev/alexlebens/helm-charts/src/branch/main/charts/postgres-cluster
+maintainers:
+  - name: alexlebens
+dependencies:
+  - name: app-template
+    alias: komodo
+    repository: https://bjw-s-labs.github.io/helm-charts/
+    version: 4.4.0
+  - name: postgres-cluster
+    alias: postgresql-17-fdb-cluster
+    version: 6.16.0
+    repository: oci://harbor.alexlebens.net/helm-charts
+icon: https://cdn.jsdelivr.net/gh/selfhst/icons/png/komodo.png
+appVersion: v1.17.5
--- a/clusters/cl01tl/helm/komodo/templates/external-secret.yaml
+++ b/clusters/cl01tl/helm/komodo/templates/external-secret.yaml
@@ -0,0 +1,116 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: komodo-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: komodo-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: passkey
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /cl01tl/komodo/config
+        metadataPolicy: None
+        property: passkey
+    - secretKey: jwt
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /cl01tl/komodo/config
+        metadataPolicy: None
+        property: jwt
+    - secretKey: webhook
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /cl01tl/komodo/config
+        metadataPolicy: None
+        property: webhook
+    - secretKey: oidc-client-id
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /authentik/oidc/komodo
+        metadataPolicy: None
+        property: client
+    - secretKey: oidc-client-secret
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /authentik/oidc/komodo
+        metadataPolicy: None
+        property: secret
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: komodo-postgresql-17-fdb-cluster-backup-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: komodo-postgresql-17-fdb-cluster-backup-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: ACCESS_KEY_ID
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /digital-ocean/home-infra/postgres-backups
+        metadataPolicy: None
+        property: access
+    - secretKey: ACCESS_SECRET_KEY
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /digital-ocean/home-infra/postgres-backups
+        metadataPolicy: None
+        property: secret
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: komodo-postgresql-17-cluster-backup-secret-garage
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: komodo-postgresql-17-cluster-backup-secret-garage
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: ACCESS_KEY_ID
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/postgres-backups
+        metadataPolicy: None
+        property: ACCESS_KEY_ID
+    - secretKey: ACCESS_SECRET_KEY
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/postgres-backups
+        metadataPolicy: None
+        property: ACCESS_SECRET_KEY
+    - secretKey: ACCESS_REGION
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/postgres-backups
+        metadataPolicy: None
+        property: ACCESS_REGION
--- a/clusters/cl01tl/helm/komodo/templates/http-route.yaml
+++ b/clusters/cl01tl/helm/komodo/templates/http-route.yaml
@@ -0,0 +1,28 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: https-route-komodo
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: https-route-komodo
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: traefik-gateway
+      namespace: traefik
+  hostnames:
+    - komodo.alexlebens.net
+  rules:
+    - matches:
+      - path:
+          type: PathPrefix
+          value: /
+      backendRefs:
+        - group: ''
+          kind: Service
+          name: komodo-main
+          port: 80
+          weight: 100
--- a/clusters/cl01tl/helm/komodo/templates/service.yaml
+++ b/clusters/cl01tl/helm/komodo/templates/service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: komodo-periphery-ps10rp
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: komodo-periphery-ps10rp
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+  annotations:
+    tailscale.com/tailnet-fqdn: komodo-periphery-ps10rp.boreal-beaufort.ts.net
+spec:
+  externalName: placeholder
+  type: ExternalName
--- a/clusters/cl01tl/helm/komodo/values.yaml
+++ b/clusters/cl01tl/helm/komodo/values.yaml
@@ -0,0 +1,236 @@
+komodo:
+  controllers:
+    main:
+      type: deployment
+      replicas: 1
+      strategy: Recreate
+      revisionHistoryLimit: 3
+      containers:
+        main:
+          image:
+            repository: ghcr.io/moghtech/komodo-core
+            tag: 1.19.5
+            pullPolicy: IfNotPresent
+          env:
+            - name: COMPOSE_LOGGING_DRIVER
+              value: local
+            - name: KOMODO_HOST
+              value: https://komodo.alexlebens.net
+            - name: KOMODO_TITLE
+              value: Komodo
+            - name: PASSKEY
+              valueFrom:
+                secretKeyRef:
+                  name: komodo-secret
+                  key: passkey
+            - name: KOMODO_MONITORING_INTERVAL
+              value: 15-sec
+            - name: KOMODO_RESOURCE_POLL_INTERVAL
+              value: 5-min
+            - name: KOMODO_PASSKEY
+              valueFrom:
+                secretKeyRef:
+                  name: komodo-secret
+                  key: passkey
+            - name: KOMODO_WEBHOOK_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: komodo-secret
+                  key: webhook
+            - name: KOMODO_JWT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: komodo-secret
+                  key: jwt
+            - name: KOMODO_LOCAL_AUTH
+              value: true
+            - name: KOMODO_ENABLE_NEW_USERS
+              value: true
+            - name: KOMODO_DISABLE_NON_ADMIN_CREATE
+              value: true
+            - name: KOMODO_TRANSPARENT_MODE
+              value: false
+            - name: PERIPHERY_SSL_ENABLED
+              value: false
+            - name: DB_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: komodo-postgresql-17-fdb-cluster-app
+                  key: user
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: komodo-postgresql-17-fdb-cluster-app
+                  key: password
+            - name: KOMODO_DATABASE_URI
+              value: mongodb://$(DB_USERNAME):$(DB_PASSWORD)@komodo-ferretdb-2.komodo:27017/komodo
+            - name: KOMODO_OIDC_ENABLED
+              value: true
+            - name: KOMODO_OIDC_PROVIDER
+              value: http://authentik-server.authentik/application/o/komodo/
+            - name: KOMODO_OIDC_REDIRECT_HOST
+              value: https://authentik.alexlebens.net
+            - name: KOMODO_OIDC_CLIENT_ID
+              valueFrom:
+                secretKeyRef:
+                  name: komodo-secret
+                  key: oidc-client-id
+            - name: KOMODO_OIDC_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: komodo-secret
+                  key: oidc-client-secret
+            - name: KOMODO_OIDC_USE_FULL_EMAIL
+              value: true
+          resources:
+            requests:
+              cpu: 10m
+              memory: 128Mi
+    ferretdb-2:
+      type: deployment
+      replicas: 1
+      strategy: Recreate
+      revisionHistoryLimit: 3
+      containers:
+        main:
+          image:
+            repository: ghcr.io/ferretdb/ferretdb
+            tag: 2.7.0
+            pullPolicy: IfNotPresent
+          env:
+            - name: FERRETDB_POSTGRESQL_URL
+              valueFrom:
+                secretKeyRef:
+                  name: komodo-postgresql-17-fdb-cluster-app
+                  key: uri
+          resources:
+            requests:
+              cpu: 10m
+              memory: 128Mi
+  service:
+    main:
+      controller: main
+      ports:
+        http:
+          port: 80
+          targetPort: 9120
+          protocol: HTTP
+    ferretdb-2:
+      controller: ferretdb-2
+      ports:
+        http:
+          port: 27017
+          targetPort: 27017
+          protocol: HTTP
+  persistence:
+    cache:
+      storageClass: ceph-block
+      accessMode: ReadWriteOnce
+      size: 5Gi
+      retain: true
+      advancedMounts:
+        main:
+          main:
+            - path: /repo-cache
+              readOnly: false
+    syncs:
+      storageClass: ceph-block
+      accessMode: ReadWriteOnce
+      size: 5Gi
+      retain: true
+      advancedMounts:
+        main:
+          main:
+            - path: /syncs
+              readOnly: false
+postgresql-17-fdb-cluster:
+  nameOverride: komodo-postgresql-17-fdb
+  mode: standalone
+  cluster:
+    image:
+      repository: ghcr.io/ferretdb/postgres-documentdb
+      tag: "17-0.106.0-ferretdb-2.5.0"
+    storage:
+      storageClass: local-path
+    walStorage:
+      storageClass: local-path
+    postgresUID: 999
+    postgresGID: 999
+    enableSuperuserAccess: true
+    monitoring:
+      enabled: true
+      prometheusRule:
+        enabled: true
+    postgresql:
+      parameters:
+        cron.database_name: 'ferretDB'
+        documentdb.enableCompact: "true"
+        documentdb.enableLetAndCollationForQueryMatch: "true"
+        documentdb.enableNowSystemVariable: "true"
+        documentdb.enableSortbyIdPushDownToPrimaryKey: "true"
+        documentdb.enableSchemaValidation: "true"
+        documentdb.enableBypassDocumentValidation: "true"
+        documentdb.enableUserCrud: "true"
+        documentdb.maxUserLimit: "100"
+        shared_buffers: 128MB
+        max_slot_wal_keep_size: 2000MB
+        hot_standby_feedback: "on"
+      pg_hba:
+        - host ferretDB postgres localhost trust
+        - host ferretDB ferret localhost trust
+      shared_preload_libraries:
+        - pg_cron
+        - pg_documentdb_core
+        - pg_documentdb
+    initdb:
+      database: ferretDB
+      owner: ferret
+      postInitApplicationSQL:
+        - create extension if not exists pg_cron;
+        - create extension if not exists documentdb cascade;
+        - grant documentdb_admin_role to ferret;
+  recovery:
+    method: objectStore
+    objectStore:
+      destinationPath: s3://postgres-backups/cl01tl/komodo/komodo-postgresql-17-fdb-cluster
+      endpointURL: http://garage-main.garage:3900
+      index: 1
+      endpointCredentials: komodo-postgresql-17-cluster-backup-secret-garage
+  backup:
+    objectStore:
+      - name: external
+        destinationPath: s3://postgres-backups-ce540ddf106d186bbddca68a/cl01tl/komodo/komodo-postgresql-17-fdb-cluster
+        index: 2
+        retentionPolicy: "30d"
+        isWALArchiver: false
+      - name: garage-local
+        destinationPath: s3://postgres-backups/cl01tl/komodo/komodo-postgresql-17-cluster
+        index: 1
+        endpointURL: http://garage-main.garage:3900
+        endpointCredentials: komodo-postgresql-17-cluster-backup-secret-garage
+        endpointCredentialsIncludeRegion: true
+        retentionPolicy: "3d"
+        isWALArchiver: true
+      # - name: garage-remote
+      #   destinationPath: s3://postgres-backups/cl01tl/komodo/komodo-postgresql-17-cluster
+      #   index: 1
+      #   endpointURL: https://garage-ps10rp.boreal-beaufort.ts.net:3900
+      #   endpointCredentials: komodo-postgresql-17-cluster-backup-secret-garage
+      #   retentionPolicy: "30d"
+      #   data:
+      #     compression: bzip2
+      #     jobs: 2
+    scheduledBackups:
+      - name: daily-backup
+        suspend: false
+        schedule: "0 0 0 * * *"
+        backupName: external
+      - name: live-backup
+        suspend: false
+        immediate: true
+        schedule: "0 0 0 * * *"
+        backupName: garage-local
+      # - name: weekly-backup
+      #   suspend: false
+      #   schedule: "0 0 4 * * SAT"
+      #   backupName: garage-remote
--- a/clusters/cl01tl/helm/kronic/Chart.lock
+++ b/clusters/cl01tl/helm/kronic/Chart.lock
@@ -0,0 +1,6 @@
+dependencies:
+- name: kronic
+  repository: https://mshade.github.io/kronic/
+  version: 0.1.7
+digest: sha256:cd9b035491c58c6fff903e2c4e750ef41e2c360555468df6a15c2457c1873fa1
+generated: "2025-12-01T19:55:52.361339-06:00"
--- a/clusters/cl01tl/helm/kronic/Chart.yaml
+++ b/clusters/cl01tl/helm/kronic/Chart.yaml
@@ -0,0 +1,22 @@
+apiVersion: v2
+name: kronic
+version: 1.0.0
+description: Kronic
+keywords:
+  - kronic
+  - cron-job
+  - dashboard
+  - kubernetes
+home: https://wiki.alexlebens.dev/s/f1191e27-264a-42bf-a3aa-3dcc35820a62
+sources:
+  - https://github.com/mshade/kronic
+  - https://github.com/mshade/kronic/pkgs/container/kronic
+  - https://github.com/mshade/kronic/tree/main/chart/kronic
+maintainers:
+  - name: alexlebens
+dependencies:
+  - name: kronic
+    repository: https://mshade.github.io/kronic/
+    version: 0.1.7
+icon: https://raw.githubusercontent.com/mshade/kronic/main/static/android-chrome-192x192.png
+appVersion:  v0.1.4
--- a/clusters/cl01tl/helm/kronic/templates/external-secret.yaml
+++ b/clusters/cl01tl/helm/kronic/templates/external-secret.yaml
@@ -0,0 +1,21 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: kronic-config-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: kronic-config-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: password
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /cl01tl/kronic/auth
+        metadataPolicy: None
+        property: password
--- a/clusters/cl01tl/helm/kronic/templates/http-route.yaml
+++ b/clusters/cl01tl/helm/kronic/templates/http-route.yaml
@@ -0,0 +1,28 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: https-route-kronic
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: https-route-kronic
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: traefik-gateway
+      namespace: traefik
+  hostnames:
+    - kronic.alexlebens.net
+  rules:
+    - matches:
+      - path:
+          type: PathPrefix
+          value: /
+      backendRefs:
+        - group: ''
+          kind: Service
+          name: kronic
+          port: 80
+          weight: 100
--- a/clusters/cl01tl/helm/kronic/values.yaml
+++ b/clusters/cl01tl/helm/kronic/values.yaml
@@ -0,0 +1,17 @@
+kronic:
+  replicaCount: 1
+  image:
+    repository: ghcr.io/mshade/kronic
+    tag: v0.1.4
+  auth:
+    enabled: true
+    adminUsername: kronic
+    existingSecretName: kronic-config-secret
+  env:
+    KRONIC_ALLOW_NAMESPACES: "gitea,vault,talos,libation,kubernetes-cloudflare-ddns"
+  ingress:
+    enabled: false
+  resources:
+    requests:
+      cpu: 10m
+      memory: 256Mi
--- a/clusters/cl01tl/helm/kube-prometheus-stack/Chart.lock
+++ b/clusters/cl01tl/helm/kube-prometheus-stack/Chart.lock
@@ -0,0 +1,9 @@
+dependencies:
+- name: kube-prometheus-stack
+  repository: oci://ghcr.io/prometheus-community/charts
+  version: 79.7.1
+- name: app-template
+  repository: https://bjw-s-labs.github.io/helm-charts/
+  version: 4.4.0
+digest: sha256:e046e7599ad195b57a8cf63b373a82d950778ac5dcc661f2ea135d433b46dacc
+generated: "2025-12-01T19:55:54.093624-06:00"
--- a/clusters/cl01tl/helm/kube-prometheus-stack/Chart.yaml
+++ b/clusters/cl01tl/helm/kube-prometheus-stack/Chart.yaml
@@ -0,0 +1,29 @@
+apiVersion: v2
+name: kube-prometheus-stack
+version: 1.0.0
+description: Kube Prometheus Stack
+keywords:
+  - kube-prometheus-stack
+  - prometheus
+  - alertmanager
+  - metrics
+  - alerts
+  - kubernetes
+home: https://wiki.alexlebens.dev/s/cd9fc3a4-aa88-4285-8886-91a6c5aecf7d
+sources:
+  - https://github.com/prometheus/prometheus
+  - https://github.com/alexbakker/alertmanager-ntfy
+  - https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack
+  - https://github.com/bjw-s/helm-charts/blob/main/charts/other/app-template
+maintainers:
+  - name: alexlebens
+dependencies:
+  - name: kube-prometheus-stack
+    version: 79.7.1
+    repository: oci://ghcr.io/prometheus-community/charts
+  - name: app-template
+    alias: ntfy-alertmanager
+    repository: https://bjw-s-labs.github.io/helm-charts/
+    version: 4.4.0
+icon: https://cdn.jsdelivr.net/gh/selfhst/icons/png/prometheus.png
+appVersion: v0.82.0
--- a/clusters/cl01tl/helm/kube-prometheus-stack/templates/external-secret.yaml
+++ b/clusters/cl01tl/helm/kube-prometheus-stack/templates/external-secret.yaml
@@ -0,0 +1,88 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: alertmanager-config-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: alertmanager-config-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: pushover_token
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /pushover/key
+        metadataPolicy: None
+        property: alertmanager_key
+    - secretKey: pushover_user_key
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /pushover/key
+        metadataPolicy: None
+        property: user_key
+    - secretKey: ntfy_password
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /cl01tl/kube-prometheus-stack/ntfy-alertmanager
+        metadataPolicy: None
+        property: ntfy_password
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: garage-metric-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: garage-metric-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: token
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/token
+        metadataPolicy: None
+        property: metric
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: ntfy-alertmanager-config-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: ntfy-alertmanager-config-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: ntfy_password
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /cl01tl/kube-prometheus-stack/ntfy-alertmanager
+        metadataPolicy: None
+        property: ntfy_password
+    - secretKey: config
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /cl01tl/kube-prometheus-stack/ntfy-alertmanager
+        metadataPolicy: None
+        property: config
--- a/clusters/cl01tl/helm/kube-prometheus-stack/templates/http-route.yaml
+++ b/clusters/cl01tl/helm/kube-prometheus-stack/templates/http-route.yaml
@@ -0,0 +1,58 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: http-route-prometheus
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: http-route-prometheus
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: traefik-gateway
+      namespace: traefik
+  hostnames:
+    - prometheus.alexlebens.net
+  rules:
+    - matches:
+      - path:
+          type: PathPrefix
+          value: /
+      backendRefs:
+        - group: ''
+          kind: Service
+          name: prometheus-operated
+          port: 9090
+          weight: 100
+
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: http-route-alertmanager
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: http-route-alertmanager
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: traefik-gateway
+      namespace: traefik
+  hostnames:
+    - alertmanager.alexlebens.net
+  rules:
+    - matches:
+      - path:
+          type: PathPrefix
+          value: /
+      backendRefs:
+        - group: ''
+          kind: Service
+          name: kube-prometheus-stack-alertmanager
+          port: 9093
+          weight: 100
--- a/clusters/cl01tl/helm/kube-prometheus-stack/templates/namespace.yaml
+++ b/clusters/cl01tl/helm/kube-prometheus-stack/templates/namespace.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kube-prometheus-stack
+  labels:
+    app.kubernetes.io/name: kube-prometheus-stack
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/warn: privileged
--- a/clusters/cl01tl/helm/kube-prometheus-stack/templates/redis-replication.yaml
+++ b/clusters/cl01tl/helm/kube-prometheus-stack/templates/redis-replication.yaml
@@ -0,0 +1,32 @@
+apiVersion: redis.redis.opstreelabs.in/v1beta2
+kind: RedisReplication
+metadata:
+  name: redis-replication-kube-prometheus-stack
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: redis-replication-kube-prometheus-stack
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  clusterSize: 3
+  podSecurityContext:
+    runAsUser: 1000
+    fsGroup: 1000
+  kubernetesConfig:
+    image: quay.io/opstree/redis:v8.0.3
+    imagePullPolicy: IfNotPresent
+    resources:
+      requests:
+        cpu: 50m
+        memory: 128Mi
+  storage:
+    volumeClaimTemplate:
+      spec:
+        storageClassName: ceph-block
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 1Gi
+  redisExporter:
+    enabled: true
+    image: quay.io/opstree/redis-exporter:v1.48.0
--- a/clusters/cl01tl/helm/kube-prometheus-stack/templates/scrape-config.yaml
+++ b/clusters/cl01tl/helm/kube-prometheus-stack/templates/scrape-config.yaml
@@ -0,0 +1,81 @@
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: ScrapeConfig
+metadata:
+  name: external-nodes-http
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: external-nodes-http
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  staticConfigs:
+    - labels:
+        job: external-nodes
+      targets:
+        - ps08rp.alexlebens.net:9100
+        - ps09rp.alexlebens.net:9100
+  metricsPath: /metrics
+  scheme: HTTP
+
+---
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: ScrapeConfig
+metadata:
+  name: external-nodes-https
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: external-nodes-https
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  staticConfigs:
+    - labels:
+        job: external-nodes
+      targets:
+        - node-exporter-ps10rp.boreal-beaufort.ts.net
+  metricsPath: /metrics
+  scheme: HTTPS
+
+---
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: ScrapeConfig
+metadata:
+  name: airgradient-http
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: airgradient-http
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  staticConfigs:
+    - labels:
+        job: airgradient
+      targets:
+        - it01ag.alexlebens.net:9926
+  metricsPath: /metrics
+  scheme: HTTP
+
+---
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: ScrapeConfig
+metadata:
+  name: garage-https
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: garage-https
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  staticConfigs:
+    - labels:
+        job: garage
+      targets:
+        - garage-ps10rp.boreal-beaufort.ts.net:3903
+  metricsPath: /metrics
+  scrapeInterval: 1m
+  scheme: HTTPS
+  authorization:
+    type: Bearer
+    credentials:
+      key: token
+      name: garage-metric-secret
--- a/clusters/cl01tl/helm/kube-prometheus-stack/templates/service-monitor.yaml
+++ b/clusters/cl01tl/helm/kube-prometheus-stack/templates/service-monitor.yaml
@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: redis-replication-kube-prometheus-stack
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: redis-replication-kube-prometheus-stack
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+    redis-operator: "true"
+    env: production
+spec:
+  selector:
+    matchLabels:
+      redis_setup_type: replication
+  endpoints:
+    - port: redis-exporter
+      interval: 30s
+      scrapeTimeout: 10s
--- a/clusters/cl01tl/helm/kube-prometheus-stack/templates/service.yaml
+++ b/clusters/cl01tl/helm/kube-prometheus-stack/templates/service.yaml
@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: node-ps10rp
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: node-ps10rp
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+  annotations:
+    tailscale.com/tailnet-fqdn: node-exporter-ps10rp.boreal-beaufort.ts.net
+spec:
+  externalName: placeholder
+  type: ExternalName
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: garage-ps10rp
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: garage-ps10rp
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+  annotations:
+    tailscale.com/tailnet-fqdn: garage-ps10rp.boreal-beaufort.ts.net
+spec:
+  externalName: placeholder
+  type: ExternalName
--- a/clusters/cl01tl/helm/kube-prometheus-stack/values.yaml
+++ b/clusters/cl01tl/helm/kube-prometheus-stack/values.yaml
@@ -0,0 +1,147 @@
+kube-prometheus-stack:
+  crds:
+    enabled: false
+  defaultRules:
+    create: true
+    rules:
+      kubeControllerManager: false
+      kubeSchedulerAlerting: false
+      kubeSchedulerRecording: false
+  global:
+    rbac:
+      create: true
+      createAggregateClusterRoles: true
+  alertmanager:
+    enabled: true
+    config:
+      route:
+        group_by: ["namespace", "alertname"]
+        group_wait: 30s
+        group_interval: 5m
+        repeat_interval: 24h
+        receiver: ntfy
+        routes:
+          - receiver: ntfy
+            group_wait: 10s
+            group_interval: 5m
+            repeat_interval: 24h
+      receivers:
+        - name: pushover
+          pushover_configs:
+            - send_resolved: true
+              user_key_file: /etc/alertmanager/secrets/alertmanager-config-secret/pushover_user_key
+              token_file: /etc/alertmanager/secrets/alertmanager-config-secret/pushover_token
+        - name: ntfy
+          webhook_configs:
+            - url: http://ntfy-alertmanager.kube-prometheus-stack:80
+              http_config:
+                basic_auth:
+                  username: ntfy-alertmanager
+                  password_file: /etc/alertmanager/secrets/alertmanager-config-secret/ntfy_password
+    alertmanagerSpec:
+      secrets:
+        - alertmanager-config-secret
+      replicas: 1
+  grafana:
+    enabled: false
+  kubeApiServer:
+    tlsConfig:
+      insecureSkipVerify: true
+  kubeControllerManager:
+    enabled: false
+  kubeEtcd:
+    enabled: true
+    service:
+      selector:
+        k8s-app: kube-controller-manager
+    serviceMonitor:
+      relabelings:
+        - sourceLabels: [__meta_kubernetes_pod_node_name]
+          separator: ;
+          regex: ^(.*)$
+          targetLabel: nodename
+          replacement: $1
+          action: replace
+      metricRelabelings:
+        - action: labeldrop
+          regex: pod
+  kubeScheduler:
+    enabled: false
+  kubeProxy:
+    enabled: false
+  kubeStateMetrics:
+    enabled: true
+  nodeExporter:
+    operatingSystems:
+      darwin:
+        enabled: false
+  prometheusOperator:
+    admissionWebhooks:
+      enabled: true
+      annotations:
+        argocd.argoproj.io/hook: PreSync
+        argocd.argoproj.io/hook-delete-policy: HookSucceeded
+      patch:
+        annotations:
+          argocd.argoproj.io/hook: PreSync
+          argocd.argoproj.io/hook-delete-policy: HookSucceeded
+      mutatingWebhookConfiguration:
+        annotations:
+          argocd.argoproj.io/hook: PreSync
+      validatingWebhookConfiguration:
+        annotations:
+          argocd.argoproj.io/hook: PreSync
+  prometheus:
+    ingress:
+      enabled: false
+    prometheusSpec:
+      scrapeInterval: 30s
+      retention: 30d
+      externalUrl: https://prometheus.alexlebens.net
+      ruleSelectorNilUsesHelmValues: false
+      serviceMonitorSelectorNilUsesHelmValues: false
+      podMonitorSelectorNilUsesHelmValues: false
+      scrapeConfigSelectorNilUsesHelmValues: false
+      storageSpec:
+        volumeClaimTemplate:
+          spec:
+            storageClassName: synology-iscsi-delete
+            accessModes: ["ReadWriteOnce"]
+            resources:
+              requests:
+                storage: 250Gi
+ntfy-alertmanager:
+  global:
+    fullnameOverride: ntfy-alertmanager
+  controllers:
+    main:
+      type: deployment
+      replicas: 1
+      strategy: Recreate
+      revisionHistoryLimit: 3
+      containers:
+        main:
+          image:
+            repository: xenrox/ntfy-alertmanager
+            tag: 0.5.0
+            pullPolicy: IfNotPresent
+  service:
+    main:
+      controller: main
+      ports:
+        http:
+          port: 80
+          targetPort: 8080
+          protocol: HTTP
+  persistence:
+    config:
+      enabled: true
+      type: secret
+      name: ntfy-alertmanager-config-secret
+      advancedMounts:
+        main:
+          main:
+            - path: /etc/ntfy-alertmanager/config
+              readOnly: true
+              mountPropagation: None
+              subPath: config
--- a/clusters/cl01tl/helm/loki/Chart.lock
+++ b/clusters/cl01tl/helm/loki/Chart.lock
@@ -0,0 +1,9 @@
+dependencies:
+- name: loki
+  repository: https://grafana.github.io/helm-charts
+  version: 6.46.0
+- name: promtail
+  repository: https://grafana.github.io/helm-charts
+  version: 6.17.1
+digest: sha256:ff5441b35309842526f15b544bf3a7cb80508b1547bbd0eb58fa584172b716be
+generated: "2025-12-01T19:55:56.473483-06:00"
--- a/clusters/cl01tl/helm/loki/Chart.yaml
+++ b/clusters/cl01tl/helm/loki/Chart.yaml
@@ -0,0 +1,25 @@
+apiVersion: v2
+name: loki
+version: 1.0.0
+description: Loki
+keywords:
+  - loki
+  - promtail
+  - logs
+  - kubernetes
+home: https://wiki.alexlebens.dev/s/c5a4dc61-5487-46c9-88f1-cac363742d30
+sources:
+  - https://github.com/grafana/loki
+  - https://github.com/grafana/loki/tree/main/production/helm/loki
+  - https://github.com/grafana/helm-charts/tree/main/charts/promtail
+maintainers:
+  - name: alexlebens
+dependencies:
+  - name: loki
+    version: 6.46.0
+    repository: https://grafana.github.io/helm-charts
+  - name: promtail
+    version: 6.17.1
+    repository: https://grafana.github.io/helm-charts
+icon: https://cdn.jsdelivr.net/gh/selfhst/icons/png/loki.png
+appVersion: 3.4.2
--- a/clusters/cl01tl/helm/loki/templates/namespace.yaml
+++ b/clusters/cl01tl/helm/loki/templates/namespace.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: loki
+  labels:
+    app.kubernetes.io/name: loki
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/warn: privileged
--- a/clusters/cl01tl/helm/loki/values.yaml
+++ b/clusters/cl01tl/helm/loki/values.yaml
@@ -0,0 +1,64 @@
+loki:
+  deploymentMode: SingleBinary
+  loki:
+    auth_enabled: false
+    commonConfig:
+      replication_factor: 1
+    ingester_client:
+      pool_config:
+        remote_timeout: 10s
+      remote_timeout: 10s
+    limits_config:
+      allow_structured_metadata: false
+      max_streams_per_user: 100000
+      ingestion_rate_mb: 1024
+      ingestion_burst_size_mb: 1024
+      retention_period: 7d
+    compactor:
+      delete_request_store: filesystem
+      working_directory: /var/loki/compactor
+      compaction_interval: 10m
+      retention_enabled: true
+      retention_delete_delay: 2h
+      retention_delete_worker_count: 150
+    storage:
+      type: filesystem
+    schemaConfig:
+      configs:
+        - from: "2024-01-11"
+          store: boltdb-shipper
+          object_store: filesystem
+          schema: v13
+          index:
+            period: 24h
+  enterprise:
+    enabled: false
+  gateway:
+    enabled: true
+    basicAuth:
+      enabled: false
+  singleBinary:
+    replicas: 1
+    persistence:
+      enableStatefulSetAutoDeletePVC: true
+      enabled: true
+      size: 150Gi
+      storageClass: synology-iscsi-delete
+  write:
+    replicas: 0
+  read:
+    replicas: 0
+  backend:
+    replicas: 0
+promtail:
+  service:
+    labels:
+      promtail: 3.0.0
+  serviceMonitor:
+    enabled: true
+    prometheusRule:
+      enabled: false
+  config:
+    clients:
+      - url: http://loki-gateway.loki.svc.cluster.local:80/loki/api/v1/push
+        tenant_id: 1
--- a/clusters/cl01tl/helm/s3-exporter/Chart.lock
+++ b/clusters/cl01tl/helm/s3-exporter/Chart.lock
@@ -0,0 +1,6 @@
+dependencies:
+- name: app-template
+  repository: https://bjw-s-labs.github.io/helm-charts/
+  version: 4.4.0
+digest: sha256:ee2fd0c7614ae0f79a67f0480e266a203ccb519c65afbd69258727ca97285f34
+generated: "2025-12-01T19:55:58.61704-06:00"
--- a/clusters/cl01tl/helm/s3-exporter/Chart.yaml
+++ b/clusters/cl01tl/helm/s3-exporter/Chart.yaml
@@ -0,0 +1,22 @@
+apiVersion: v2
+name: s3-exporter
+version: 1.0.0
+description: S3 Exporter
+keywords:
+  - s3-exporter
+  - storage
+  - monitoring
+  - metrics
+home: https://wiki.alexlebens.dev/s/
+sources:
+  - https://github.com/molu8bits/s3bucket_exporter
+  - https://hub.docker.com/r/molu8bits/s3bucket_exporter
+  - https://github.com/bjw-s/helm-charts/blob/main/charts/other/app-template
+maintainers:
+  - name: alexlebens
+dependencies:
+  - name: app-template
+    alias: s3-exporter
+    repository: https://bjw-s-labs.github.io/helm-charts/
+    version: 4.4.0
+appVersion: 1.0.2
--- a/clusters/cl01tl/helm/s3-exporter/templates/external-secret.yaml
+++ b/clusters/cl01tl/helm/s3-exporter/templates/external-secret.yaml
@@ -0,0 +1,102 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: s3-do-home-infra-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: s3-do-home-infra-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: AWS_ACCESS_KEY_ID
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /digital-ocean/home-infra/all-access
+        metadataPolicy: None
+        property: AWS_ACCESS_KEY_ID
+    - secretKey: AWS_SECRET_ACCESS_KEY
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /digital-ocean/home-infra/all-access
+        metadataPolicy: None
+        property: AWS_SECRET_ACCESS_KEY
+    - secretKey: AWS_REGION
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /digital-ocean/home-infra/prometheus-exporter
+        metadataPolicy: None
+        property: AWS_REGION
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: s3-ceph-directus-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: s3-ceph-directus-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: AWS_ACCESS_KEY_ID
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /cl01tl/directus/ceph
+        metadataPolicy: None
+        property: AWS_ACCESS_KEY_ID
+    - secretKey: AWS_SECRET_ACCESS_KEY
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /cl01tl/directus/ceph
+        metadataPolicy: None
+        property: AWS_SECRET_ACCESS_KEY
+    - secretKey: BUCKET_HOST
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /cl01tl/directus/ceph
+        metadataPolicy: None
+        property: BUCKET_HOST
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: s3-garage-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: s3-garage-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: AWS_ACCESS_KEY_ID
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/s3-exporter
+        metadataPolicy: None
+        property: ACCESS_KEY_ID
+    - secretKey: AWS_SECRET_ACCESS_KEY
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/s3-exporter
+        metadataPolicy: None
+        property: ACCESS_SECRET_KEY
--- a/clusters/cl01tl/helm/s3-exporter/templates/service-monitor.yaml
+++ b/clusters/cl01tl/helm/s3-exporter/templates/service-monitor.yaml
@@ -0,0 +1,86 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: s3-exporter-digital-ocean
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: s3-exporter-digital-ocean
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: s3-exporter
+      app.kubernetes.io/instance: s3-exporter
+      app.kubernetes.io/service: s3-exporter-digital-ocean
+  endpoints:
+    - port: metrics
+      interval: 5m
+      scrapeTimeout: 120s
+      path: /metrics
+
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: s3-exporter-ceph-directus
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: s3-exporter-ceph-directus
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: s3-exporter
+      app.kubernetes.io/instance: s3-exporter
+      app.kubernetes.io/service: s3-exporter-ceph-directus
+  endpoints:
+    - port: metrics
+      interval: 5m
+      scrapeTimeout: 120s
+      path: /metrics
+
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: s3-exporter-garage-local
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: s3-exporter-garage-local
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: s3-exporter
+      app.kubernetes.io/instance: s3-exporter
+      app.kubernetes.io/service: s3-exporter-garage-local
+  endpoints:
+    - port: metrics
+      interval: 5m
+      scrapeTimeout: 120s
+      path: /metrics
+
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: s3-exporter-garage-remote
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: s3-exporter-garage-remote
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: s3-exporter
+      app.kubernetes.io/instance: s3-exporter
+      app.kubernetes.io/service: s3-exporter-garage-remote
+  endpoints:
+    - port: metrics
+      interval: 5m
+      scrapeTimeout: 120s
+      path: /metrics
--- a/clusters/cl01tl/helm/s3-exporter/values.yaml
+++ b/clusters/cl01tl/helm/s3-exporter/values.yaml
@@ -0,0 +1,181 @@
+s3-exporter:
+  controllers:
+    digital-ocean:
+      type: deployment
+      replicas: 1
+      strategy: Recreate
+      revisionHistoryLimit: 3
+      containers:
+        main:
+          image:
+            repository: molu8bits/s3bucket_exporter
+            tag: 1.0.2
+            pullPolicy: IfNotPresent
+          env:
+            - name: S3_NAME
+              value: digital-ocean
+            - name: S3_ENDPOINT
+              value: https://nyc3.digitaloceanspaces.com
+            - name: S3_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: s3-do-home-infra-secret
+                  key: AWS_ACCESS_KEY_ID
+            - name: S3_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: s3-do-home-infra-secret
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: S3_REGION
+              valueFrom:
+                secretKeyRef:
+                  name: s3-do-home-infra-secret
+                  key: AWS_REGION
+            - name: LOG_LEVEL
+              value: info
+            - name: S3_FORCE_PATH_STYLE
+              value: false
+          resources:
+            requests:
+              cpu: 10m
+              memory: 64Mi
+    ceph-directus:
+      type: deployment
+      replicas: 1
+      strategy: Recreate
+      revisionHistoryLimit: 3
+      containers:
+        main:
+          image:
+            repository: molu8bits/s3bucket_exporter
+            tag: 1.0.2
+            pullPolicy: IfNotPresent
+          env:
+            - name: S3_NAME
+              value: ceph-directus
+            - name: S3_ENDPOINT
+              valueFrom:
+                secretKeyRef:
+                  name: s3-ceph-directus-secret
+                  key: BUCKET_HOST
+            - name: S3_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: s3-ceph-directus-secret
+                  key: AWS_ACCESS_KEY_ID
+            - name: S3_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: s3-ceph-directus-secret
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: S3_REGION
+              value: us-east-1
+            - name: LOG_LEVEL
+              value: info
+            - name: S3_FORCE_PATH_STYLE
+              value: true
+          resources:
+            requests:
+              cpu: 10m
+              memory: 64Mi
+    garage-local:
+      type: deployment
+      replicas: 1
+      strategy: Recreate
+      revisionHistoryLimit: 3
+      containers:
+        main:
+          image:
+            repository: molu8bits/s3bucket_exporter
+            tag: 1.0.2
+            pullPolicy: IfNotPresent
+          env:
+            - name: S3_NAME
+              value: garage-local
+            - name: S3_ENDPOINT
+              value: http://garage-main.garage:3900
+            - name: S3_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: s3-garage-secret
+                  key: AWS_ACCESS_KEY_ID
+            - name: S3_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: s3-garage-secret
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: S3_REGION
+              value: us-east-1
+            - name: LOG_LEVEL
+              value: debug
+            - name: S3_FORCE_PATH_STYLE
+              value: true
+          resources:
+            requests:
+              cpu: 10m
+              memory: 64Mi
+    garage-remote:
+      type: deployment
+      replicas: 1
+      strategy: Recreate
+      revisionHistoryLimit: 3
+      containers:
+        main:
+          image:
+            repository: molu8bits/s3bucket_exporter
+            tag: 1.0.2
+            pullPolicy: IfNotPresent
+          env:
+            - name: S3_NAME
+              value: garage-remote
+            - name: S3_ENDPOINT
+              value: https://garage-ps10rp.boreal-beaufort.ts.net:3900
+            - name: S3_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: s3-garage-secret
+                  key: AWS_ACCESS_KEY_ID
+            - name: S3_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: s3-garage-secret
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: S3_REGION
+              value: us-east-1
+            - name: LOG_LEVEL
+              value: debug
+            - name: S3_FORCE_PATH_STYLE
+              value: true
+          resources:
+            requests:
+              cpu: 10m
+              memory: 64Mi
+  service:
+    digital-ocean:
+      controller: digital-ocean
+      ports:
+        metrics:
+          port: 9655
+          targetPort: 9655
+          protocol: TCP
+    ceph-directus:
+      controller: ceph-directus
+      ports:
+        metrics:
+          port: 9655
+          targetPort: 9655
+          protocol: TCP
+    garage-local:
+      controller: garage-local
+      ports:
+        metrics:
+          port: 9655
+          targetPort: 9655
+          protocol: TCP
+    garage-remote:
+      controller: garage-remote
+      ports:
+        metrics:
+          port: 9655
+          targetPort: 9655
+          protocol: TCP
--- a/clusters/cl01tl/helm/shelly-plug/Chart.lock
+++ b/clusters/cl01tl/helm/shelly-plug/Chart.lock
@@ -0,0 +1,6 @@
+dependencies:
+- name: app-template
+  repository: https://bjw-s-labs.github.io/helm-charts/
+  version: 4.4.0
+digest: sha256:8fd52e5094f007d59b3a7544c3367f2ad9826acbff17d0972dcd68585a0334e4
+generated: "2025-12-01T19:56:00.069183-06:00"
--- a/clusters/cl01tl/helm/shelly-plug/Chart.yaml
+++ b/clusters/cl01tl/helm/shelly-plug/Chart.yaml
@@ -0,0 +1,20 @@
+apiVersion: v2
+name: shelly-plug
+version: 1.0.0
+description: Shelly Plug
+keywords:
+  - shelly-plug
+  - metrics
+home: https://wiki.alexlebens.dev/s/18b5575c-3a57-4515-89a0-b23d6df8dec4
+sources:
+  - https://github.com/geerlingguy/shelly-plug-prometheus
+  - https://hub.docker.com/_/php
+  - https://github.com/bjw-s/helm-charts/blob/main/charts/other/app-template/values.yaml
+maintainers:
+  - name: alexlebens
+dependencies:
+  - name: app-template
+    alias: shelly-plug
+    repository: https://bjw-s-labs.github.io/helm-charts/
+    version: 4.4.0
+appVersion: 1.0.0
--- a/clusters/cl01tl/helm/shelly-plug/templates/external-secret.yaml
+++ b/clusters/cl01tl/helm/shelly-plug/templates/external-secret.yaml
@@ -0,0 +1,28 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: shelly-plug-config-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: shelly-plug-config-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: SHELLY_HTTP_USERNAME
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /shelly-plug/auth/it05sp
+        metadataPolicy: None
+        property: SHELLY_HTTP_USERNAME
+    - secretKey: SHELLY_HTTP_PASSWORD
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /shelly-plug/auth/it05sp
+        metadataPolicy: None
+        property: SHELLY_HTTP_PASSWORD
--- a/clusters/cl01tl/helm/shelly-plug/templates/service-monitor.yaml
+++ b/clusters/cl01tl/helm/shelly-plug/templates/service-monitor.yaml
@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: shelly-plug
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: shelly-plug
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: shelly-plug
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  endpoints:
+    - port: metrics
+      interval: 30s
+      scrapeTimeout: 10s
+      path: /metrics
--- a/clusters/cl01tl/helm/shelly-plug/values.yaml
+++ b/clusters/cl01tl/helm/shelly-plug/values.yaml
@@ -0,0 +1,74 @@
+shelly-plug:
+  controllers:
+    main:
+      type: deployment
+      replicas: 1
+      strategy: Recreate
+      revisionHistoryLimit: 3
+      initContainers:
+        init-fetch-repo:
+          securityContext:
+            runAsUser: 0
+          image:
+            repository: alpine/git
+            tag: latest
+            pullPolicy: IfNotPresent
+          command:
+            - /bin/sh
+            - -ec
+            - |
+              cd /var/www/html
+              if [ -d ".git" ]; then
+                echo "Git repository found. Pulling latest changes..."
+                git pull
+              else
+                echo "Not a git repository. Initializing ..."
+                git init
+                git remote add origin https://github.com/geerlingguy/shelly-plug-prometheus.git
+                git fetch origin
+                git checkout origin/master -ft
+              fi
+          resources:
+            requests:
+              cpu: 10m
+              memory: 128Mi
+      containers:
+        main:
+          image:
+            repository: php
+            tag: 8.4.15-apache-bookworm
+            pullPolicy: IfNotPresent
+          env:
+            - name: SHELLY_HOSTNAME
+              value: it05sp.alexlebens.net
+            - name: SHELLY_GENERATION
+              value: 2
+          envFrom:
+            - secretRef:
+                name: shelly-plug-config-secret
+          resources:
+            requests:
+              cpu: 10m
+              memory: 64Mi
+  service:
+    main:
+      controller: main
+      ports:
+        metrics:
+          port: 80
+          targetPort: 80
+          protocol: TCP
+  persistence:
+    script:
+      storageClass: ceph-block
+      accessMode: ReadWriteOnce
+      size: 1Gi
+      retain: true
+      advancedMounts:
+        main:
+          init-fetch-repo:
+            - path: /var/www/html
+              readOnly: false
+          main:
+            - path: /var/www/html
+              readOnly: false
--- a/clusters/cl01tl/helm/trivy/Chart.lock
+++ b/clusters/cl01tl/helm/trivy/Chart.lock
@@ -0,0 +1,6 @@
+dependencies:
+- name: trivy-operator
+  repository: https://aquasecurity.github.io/helm-charts/
+  version: 0.31.0
+digest: sha256:5a71d5ff43d5e36ea500c5dcade70cddc874621ad49ffe7c10ba202a14b9c87f
+generated: "2025-12-01T19:56:01.513264-06:00"
--- a/clusters/cl01tl/helm/trivy/Chart.yaml
+++ b/clusters/cl01tl/helm/trivy/Chart.yaml
@@ -0,0 +1,22 @@
+apiVersion: v2
+name: trivy
+version: 1.0.0
+description: Trivy
+keywords:
+  - trivy
+  - vulnerability
+  - monitoring
+  - kubernetes
+home: https://wiki.alexlebens.dev/s/5cffa529-4c2e-4126-99eb-cc4aeb5a49b3
+sources:
+  - https://github.com/aquasecurity/trivy
+  - https://github.com/aquasecurity/trivy-operator
+  - https://github.com/aquasecurity/trivy-operator/tree/main/deploy/helm
+maintainers:
+  - name: alexlebens
+dependencies:
+  - name: trivy-operator
+    version: 0.31.0
+    repository: https://aquasecurity.github.io/helm-charts/
+icon: https://raw.githubusercontent.com/aquasecurity/trivy/main/docs/imgs/logo.png
+appVersion: v0.26.1
--- a/clusters/cl01tl/helm/trivy/templates/namespace.yaml
+++ b/clusters/cl01tl/helm/trivy/templates/namespace.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: trivy
+  labels:
+    app.kubernetes.io/name: trivy
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/warn: privileged
--- a/clusters/cl01tl/helm/trivy/values.yaml
+++ b/clusters/cl01tl/helm/trivy/values.yaml
@@ -0,0 +1,105 @@
+trivy-operator:
+  targetWorkloads: "pod,replicaset,replicationcontroller,statefulset,daemonset,cronjob,job"
+  operator:
+    replicas: 1
+    vulnerabilityScannerEnabled: false
+    sbomGenerationEnabled: false
+    clusterSbomCacheEnabled: false
+    configAuditScannerEnabled: true
+    rbacAssessmentScannerEnabled: true
+    infraAssessmentScannerEnabled: false
+    clusterComplianceEnabled: false
+  serviceMonitor:
+    enabled: true
+  trivy:
+    createConfig: true
+    image:
+      registry: mirror.gcr.io
+      repository: aquasec/trivy
+      tag: 0.67.2
+    storageClassEnabled: true
+    storageClassName: ceph-block
+    storageSize: "5Gi"
+    registry:
+      mirror:
+        "registry-1.docker.io": proxy-registry-1.docker.io
+        "quay.io": proxy-quay.io
+        "registry.k8s.io": proxy-registry.k8s
+        "gcr.io": proxy-gcr.io
+        "ghcr.io": proxy-ghcr.io
+        "hub.docker": proxy-hub.docker
+    severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL
+    slow: true
+    resources:
+      requests:
+        cpu: 100m
+        memory: 128M
+    supportedConfigAuditKinds: "Workload,Service,Role,ClusterRole,NetworkPolicy,Ingress,LimitRange,ResourceQuota"
+    server:
+      resources:
+        requests:
+          cpu: 200m
+          memory: 512Mi
+      replicas: 1
+  compliance:
+    reportType: summary
+    cron: 0 5 * * *
+    specs:
+      - k8s-cis-1.23
+      - k8s-nsa-1.0
+      - k8s-pss-baseline-0.1
+      - k8s-pss-restricted-0.1
+  volumeMounts:
+    - mountPath: /tmp
+      name: cache-policies
+      readOnly: false
+  volumes:
+    - name: cache-policies
+      emptyDir: {}
+  resources:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+  nodeCollector:
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+    volumeMounts:
+      - name: var-lib-etcd
+        mountPath: /var/lib/etcd
+        readOnly: true
+      - name: var-lib-kubelet
+        mountPath: /var/lib/kubelet
+        readOnly: true
+      - name: var-lib-kube-scheduler
+        mountPath: /var/lib/kube-scheduler
+        readOnly: true
+      - name: var-lib-kube-controller-manager
+        mountPath: /var/lib/kube-controller-manager
+        readOnly: true
+      - name: etc-kubernetes
+        mountPath: /etc/kubernetes
+        readOnly: true
+      - name: etc-cni-netd
+        mountPath: /etc/cni/net.d/
+        readOnly: true
+    volumes:
+      - name: var-lib-etcd
+        hostPath:
+          path: /var/lib/etcd
+      - name: var-lib-kubelet
+        hostPath:
+          path: /var/lib/kubelet
+      - name: var-lib-kube-scheduler
+        hostPath:
+          path: /var/lib/kube-scheduler
+      - name: var-lib-kube-controller-manager
+        hostPath:
+          path: /var/lib/kube-controller-manager
+      - name: etc-kubernetes
+        hostPath:
+          path: /etc/kubernetes
+      - name: etc-cni-netd
+        hostPath:
+          path: /etc/cni/net.d/
--- a/clusters/cl01tl/helm/unpoller/Chart.lock
+++ b/clusters/cl01tl/helm/unpoller/Chart.lock
@@ -0,0 +1,6 @@
+dependencies:
+- name: app-template
+  repository: https://bjw-s-labs.github.io/helm-charts/
+  version: 4.4.0
+digest: sha256:7ccecd4e314f77652088644bc235465b4317c7afd3736a623c484d2b296eefa2
+generated: "2025-12-01T19:56:03.244251-06:00"
--- a/clusters/cl01tl/helm/unpoller/Chart.yaml
+++ b/clusters/cl01tl/helm/unpoller/Chart.yaml
@@ -0,0 +1,23 @@
+apiVersion: v2
+name: unpoller
+version: 1.0.0
+description: Unpoller
+keywords:
+  - unpoller
+  - ubiquiti
+  - unifi
+  - metrics
+home: https://wiki.alexlebens.dev/s/cac4e7b1-3d8e-4a32-993c-c6b3f1d2c344
+sources:
+  - https://github.com/unpoller/unpoller
+  - https://github.com/unpoller/unpoller/pkgs/container/unpoller
+  - https://github.com/bjw-s/helm-charts/blob/main/charts/other/app-template/values.yaml
+maintainers:
+  - name: alexlebens
+dependencies:
+  - name: app-template
+    alias: unpoller
+    repository: https://bjw-s-labs.github.io/helm-charts/
+    version: 4.4.0
+icon: https://camo.githubusercontent.com/c5d07a5b3acfeac8e1c25bf56f440ffe032b86e4e7f15de82357f022a43fc927/68747470733a2f2f756e706f6c6c65722e636f6d2f696d672f6c6f676f2e706e67
+appVersion: v2.15.3
--- a/clusters/cl01tl/helm/unpoller/templates/external-secret.yaml
+++ b/clusters/cl01tl/helm/unpoller/templates/external-secret.yaml
@@ -0,0 +1,28 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: unpoller-unifi-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: unpoller-unifi-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: UP_UNIFI_CONTROLLER_0_USER
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /unifi/auth/cl01tl
+        metadataPolicy: None
+        property: user
+    - secretKey: UP_UNIFI_CONTROLLER_0_PASS
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /unifi/auth/cl01tl
+        metadataPolicy: None
+        property: password
--- a/clusters/cl01tl/helm/unpoller/templates/service-monitor.yaml
+++ b/clusters/cl01tl/helm/unpoller/templates/service-monitor.yaml
@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: unpoller
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: unpoller
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: unpoller
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  endpoints:
+    - port: metrics
+      interval: 30s
+      scrapeTimeout: 10s
+      path: /metrics
--- a/clusters/cl01tl/helm/unpoller/values.yaml
+++ b/clusters/cl01tl/helm/unpoller/values.yaml
@@ -0,0 +1,55 @@
+unpoller:
+  controllers:
+    main:
+      type: deployment
+      replicas: 1
+      strategy: Recreate
+      revisionHistoryLimit: 3
+      containers:
+        main:
+          image:
+            repository: ghcr.io/unpoller/unpoller
+            tag: v2.15.4
+            pullPolicy: IfNotPresent
+          env:
+            - name: UP_UNIFI_CONTROLLER_0_SAVE_ALARMS
+              value: 'false'
+            - name: UP_UNIFI_CONTROLLER_0_SAVE_ANOMALIES
+              value: 'false'
+            - name: UP_UNIFI_CONTROLLER_0_SAVE_DPI
+              value: 'false'
+            - name: UP_UNIFI_CONTROLLER_0_SAVE_EVENTS
+              value: 'false'
+            - name: UP_UNIFI_CONTROLLER_0_SAVE_IDS
+              value: 'false'
+            - name: UP_UNIFI_CONTROLLER_0_SAVE_SITES
+              value: 'true'
+            - name: UP_UNIFI_CONTROLLER_0_URL
+              value: https://unifi.alexlebens.net/
+            - name: UP_UNIFI_CONTROLLER_0_VERIFY_SSL
+              value: 'false'
+            - name: UP_INFLUXDB_DISABLE
+              value: 'true'
+            - name: UP_PROMETHEUS_HTTP_LISTEN
+              value: 0.0.0.0:9130
+            - name: UP_PROMETHEUS_NAMESPACE
+              value: unpoller
+            - name: UP_POLLER_DEBUG
+              value: 'false'
+            - name: UP_POLLER_QUIET
+              value: 'false'
+          envFrom:
+            - secretRef:
+                name: unpoller-unifi-secret
+          resources:
+            requests:
+              cpu: 10m
+              memory: 64Mi
+  service:
+    main:
+      controller: main
+      ports:
+        metrics:
+          port: 9130
+          targetPort: 9130
+          protocol: TCP