migrate

clusters/cl01tl/helm/kube-prometheus-stack/Chart.lock

@@ -0,0 +1,9 @@
+dependencies:
+- name: kube-prometheus-stack
+  repository: oci://ghcr.io/prometheus-community/charts
+  version: 79.7.1
+- name: app-template
+  repository: https://bjw-s-labs.github.io/helm-charts/
+  version: 4.4.0
+digest: sha256:e046e7599ad195b57a8cf63b373a82d950778ac5dcc661f2ea135d433b46dacc
+generated: "2025-12-01T19:55:54.093624-06:00"

clusters/cl01tl/helm/kube-prometheus-stack/Chart.yaml

@@ -0,0 +1,29 @@
+apiVersion: v2
+name: kube-prometheus-stack
+version: 1.0.0
+description: Kube Prometheus Stack
+keywords:
+  - kube-prometheus-stack
+  - prometheus
+  - alertmanager
+  - metrics
+  - alerts
+  - kubernetes
+home: https://wiki.alexlebens.dev/s/cd9fc3a4-aa88-4285-8886-91a6c5aecf7d
+sources:
+  - https://github.com/prometheus/prometheus
+  - https://github.com/alexbakker/alertmanager-ntfy
+  - https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack
+  - https://github.com/bjw-s/helm-charts/blob/main/charts/other/app-template
+maintainers:
+  - name: alexlebens
+dependencies:
+  - name: kube-prometheus-stack
+    version: 79.7.1
+    repository: oci://ghcr.io/prometheus-community/charts
+  - name: app-template
+    alias: ntfy-alertmanager
+    repository: https://bjw-s-labs.github.io/helm-charts/
+    version: 4.4.0
+icon: https://cdn.jsdelivr.net/gh/selfhst/icons/png/prometheus.png
+appVersion: v0.82.0

clusters/cl01tl/helm/kube-prometheus-stack/templates/external-secret.yaml

@@ -0,0 +1,88 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: alertmanager-config-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: alertmanager-config-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: pushover_token
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /pushover/key
+        metadataPolicy: None
+        property: alertmanager_key
+    - secretKey: pushover_user_key
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /pushover/key
+        metadataPolicy: None
+        property: user_key
+    - secretKey: ntfy_password
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /cl01tl/kube-prometheus-stack/ntfy-alertmanager
+        metadataPolicy: None
+        property: ntfy_password
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: garage-metric-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: garage-metric-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: token
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/token
+        metadataPolicy: None
+        property: metric
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: ntfy-alertmanager-config-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: ntfy-alertmanager-config-secret
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: ntfy_password
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /cl01tl/kube-prometheus-stack/ntfy-alertmanager
+        metadataPolicy: None
+        property: ntfy_password
+    - secretKey: config
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /cl01tl/kube-prometheus-stack/ntfy-alertmanager
+        metadataPolicy: None
+        property: config

clusters/cl01tl/helm/kube-prometheus-stack/templates/http-route.yaml

@@ -0,0 +1,58 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: http-route-prometheus
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: http-route-prometheus
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: traefik-gateway
+      namespace: traefik
+  hostnames:
+    - prometheus.alexlebens.net
+  rules:
+    - matches:
+      - path:
+          type: PathPrefix
+          value: /
+      backendRefs:
+        - group: ''
+          kind: Service
+          name: prometheus-operated
+          port: 9090
+          weight: 100
+
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: http-route-alertmanager
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: http-route-alertmanager
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: traefik-gateway
+      namespace: traefik
+  hostnames:
+    - alertmanager.alexlebens.net
+  rules:
+    - matches:
+      - path:
+          type: PathPrefix
+          value: /
+      backendRefs:
+        - group: ''
+          kind: Service
+          name: kube-prometheus-stack-alertmanager
+          port: 9093
+          weight: 100

clusters/cl01tl/helm/kube-prometheus-stack/templates/namespace.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kube-prometheus-stack
+  labels:
+    app.kubernetes.io/name: kube-prometheus-stack
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/warn: privileged

clusters/cl01tl/helm/kube-prometheus-stack/templates/redis-replication.yaml

@@ -0,0 +1,32 @@
+apiVersion: redis.redis.opstreelabs.in/v1beta2
+kind: RedisReplication
+metadata:
+  name: redis-replication-kube-prometheus-stack
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: redis-replication-kube-prometheus-stack
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  clusterSize: 3
+  podSecurityContext:
+    runAsUser: 1000
+    fsGroup: 1000
+  kubernetesConfig:
+    image: quay.io/opstree/redis:v8.0.3
+    imagePullPolicy: IfNotPresent
+    resources:
+      requests:
+        cpu: 50m
+        memory: 128Mi
+  storage:
+    volumeClaimTemplate:
+      spec:
+        storageClassName: ceph-block
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 1Gi
+  redisExporter:
+    enabled: true
+    image: quay.io/opstree/redis-exporter:v1.48.0

clusters/cl01tl/helm/kube-prometheus-stack/templates/scrape-config.yaml

@@ -0,0 +1,81 @@
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: ScrapeConfig
+metadata:
+  name: external-nodes-http
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: external-nodes-http
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  staticConfigs:
+    - labels:
+        job: external-nodes
+      targets:
+        - ps08rp.alexlebens.net:9100
+        - ps09rp.alexlebens.net:9100
+  metricsPath: /metrics
+  scheme: HTTP
+
+---
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: ScrapeConfig
+metadata:
+  name: external-nodes-https
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: external-nodes-https
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  staticConfigs:
+    - labels:
+        job: external-nodes
+      targets:
+        - node-exporter-ps10rp.boreal-beaufort.ts.net
+  metricsPath: /metrics
+  scheme: HTTPS
+
+---
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: ScrapeConfig
+metadata:
+  name: airgradient-http
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: airgradient-http
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  staticConfigs:
+    - labels:
+        job: airgradient
+      targets:
+        - it01ag.alexlebens.net:9926
+  metricsPath: /metrics
+  scheme: HTTP
+
+---
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: ScrapeConfig
+metadata:
+  name: garage-https
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: garage-https
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  staticConfigs:
+    - labels:
+        job: garage
+      targets:
+        - garage-ps10rp.boreal-beaufort.ts.net:3903
+  metricsPath: /metrics
+  scrapeInterval: 1m
+  scheme: HTTPS
+  authorization:
+    type: Bearer
+    credentials:
+      key: token
+      name: garage-metric-secret

clusters/cl01tl/helm/kube-prometheus-stack/templates/service-monitor.yaml

@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: redis-replication-kube-prometheus-stack
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: redis-replication-kube-prometheus-stack
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+    redis-operator: "true"
+    env: production
+spec:
+  selector:
+    matchLabels:
+      redis_setup_type: replication
+  endpoints:
+    - port: redis-exporter
+      interval: 30s
+      scrapeTimeout: 10s

clusters/cl01tl/helm/kube-prometheus-stack/templates/service.yaml

@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: node-ps10rp
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: node-ps10rp
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+  annotations:
+    tailscale.com/tailnet-fqdn: node-exporter-ps10rp.boreal-beaufort.ts.net
+spec:
+  externalName: placeholder
+  type: ExternalName
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: garage-ps10rp
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: garage-ps10rp
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+  annotations:
+    tailscale.com/tailnet-fqdn: garage-ps10rp.boreal-beaufort.ts.net
+spec:
+  externalName: placeholder
+  type: ExternalName

clusters/cl01tl/helm/kube-prometheus-stack/values.yaml

@@ -0,0 +1,147 @@
+kube-prometheus-stack:
+  crds:
+    enabled: false
+  defaultRules:
+    create: true
+    rules:
+      kubeControllerManager: false
+      kubeSchedulerAlerting: false
+      kubeSchedulerRecording: false
+  global:
+    rbac:
+      create: true
+      createAggregateClusterRoles: true
+  alertmanager:
+    enabled: true
+    config:
+      route:
+        group_by: ["namespace", "alertname"]
+        group_wait: 30s
+        group_interval: 5m
+        repeat_interval: 24h
+        receiver: ntfy
+        routes:
+          - receiver: ntfy
+            group_wait: 10s
+            group_interval: 5m
+            repeat_interval: 24h
+      receivers:
+        - name: pushover
+          pushover_configs:
+            - send_resolved: true
+              user_key_file: /etc/alertmanager/secrets/alertmanager-config-secret/pushover_user_key
+              token_file: /etc/alertmanager/secrets/alertmanager-config-secret/pushover_token
+        - name: ntfy
+          webhook_configs:
+            - url: http://ntfy-alertmanager.kube-prometheus-stack:80
+              http_config:
+                basic_auth:
+                  username: ntfy-alertmanager
+                  password_file: /etc/alertmanager/secrets/alertmanager-config-secret/ntfy_password
+    alertmanagerSpec:
+      secrets:
+        - alertmanager-config-secret
+      replicas: 1
+  grafana:
+    enabled: false
+  kubeApiServer:
+    tlsConfig:
+      insecureSkipVerify: true
+  kubeControllerManager:
+    enabled: false
+  kubeEtcd:
+    enabled: true
+    service:
+      selector:
+        k8s-app: kube-controller-manager
+    serviceMonitor:
+      relabelings:
+        - sourceLabels: [__meta_kubernetes_pod_node_name]
+          separator: ;
+          regex: ^(.*)$
+          targetLabel: nodename
+          replacement: $1
+          action: replace
+      metricRelabelings:
+        - action: labeldrop
+          regex: pod
+  kubeScheduler:
+    enabled: false
+  kubeProxy:
+    enabled: false
+  kubeStateMetrics:
+    enabled: true
+  nodeExporter:
+    operatingSystems:
+      darwin:
+        enabled: false
+  prometheusOperator:
+    admissionWebhooks:
+      enabled: true
+      annotations:
+        argocd.argoproj.io/hook: PreSync
+        argocd.argoproj.io/hook-delete-policy: HookSucceeded
+      patch:
+        annotations:
+          argocd.argoproj.io/hook: PreSync
+          argocd.argoproj.io/hook-delete-policy: HookSucceeded
+      mutatingWebhookConfiguration:
+        annotations:
+          argocd.argoproj.io/hook: PreSync
+      validatingWebhookConfiguration:
+        annotations:
+          argocd.argoproj.io/hook: PreSync
+  prometheus:
+    ingress:
+      enabled: false
+    prometheusSpec:
+      scrapeInterval: 30s
+      retention: 30d
+      externalUrl: https://prometheus.alexlebens.net
+      ruleSelectorNilUsesHelmValues: false
+      serviceMonitorSelectorNilUsesHelmValues: false
+      podMonitorSelectorNilUsesHelmValues: false
+      scrapeConfigSelectorNilUsesHelmValues: false
+      storageSpec:
+        volumeClaimTemplate:
+          spec:
+            storageClassName: synology-iscsi-delete
+            accessModes: ["ReadWriteOnce"]
+            resources:
+              requests:
+                storage: 250Gi
+ntfy-alertmanager:
+  global:
+    fullnameOverride: ntfy-alertmanager
+  controllers:
+    main:
+      type: deployment
+      replicas: 1
+      strategy: Recreate
+      revisionHistoryLimit: 3
+      containers:
+        main:
+          image:
+            repository: xenrox/ntfy-alertmanager
+            tag: 0.5.0
+            pullPolicy: IfNotPresent
+  service:
+    main:
+      controller: main
+      ports:
+        http:
+          port: 80
+          targetPort: 8080
+          protocol: HTTP
+  persistence:
+    config:
+      enabled: true
+      type: secret
+      name: ntfy-alertmanager-config-secret
+      advancedMounts:
+        main:
+          main:
+            - path: /etc/ntfy-alertmanager/config
+              readOnly: true
+              mountPropagation: None
+              subPath: config