diff --git a/clusters/cl01tl/manifests/cloudnative-pg/CronJob-cloudnative-pg-postgres-backups-external-rclone.yaml b/clusters/cl01tl/manifests/cloudnative-pg/CronJob-cloudnative-pg-postgres-backups-external-rclone.yaml index d521252b2..7d704ab10 100644 --- a/clusters/cl01tl/manifests/cloudnative-pg/CronJob-cloudnative-pg-postgres-backups-external-rclone.yaml +++ b/clusters/cl01tl/manifests/cloudnative-pg/CronJob-cloudnative-pg-postgres-backups-external-rclone.yaml @@ -7,8 +7,8 @@ metadata: app.kubernetes.io/instance: cloudnative-pg app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: postgres-backups-external-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-postgres-backups-external-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-postgres-backups-external-0.11.0 namespace: cloudnative-pg spec: suspend: false @@ -46,7 +46,7 @@ spec: - --verbose - --rc - --rc-enable-metrics - - --metrics-addr=0.0.0.0:5572 + - --metrics-addr=0.0.0.0:5574 env: - name: RCLONE_CONFIG_DEST_TYPE value: s3 @@ -76,7 +76,7 @@ spec: name: postgres-backups-775957147abfbc73-rclone-destination-config - name: RCLONE_CONFIG_DEST_FORCE_PATH_STYLE value: "true" - image: rclone/rclone:1.74.0@sha256:d2e0e88359d0b2e67cfcd2c43d5405185eb8adfc207079df27c42da82c5207bc + image: rclone/rclone:1.74.1@sha256:eb8788b795f0009324e9572b0e2acb9b56885327c2746b07d67a9d3b893a6602 imagePullPolicy: IfNotPresent name: prune - args: @@ -92,7 +92,7 @@ spec: - '**/walls/**' - --rc - --rc-enable-metrics - - --metrics-addr=0.0.0.0:5572 + - --metrics-addr=0.0.0.0:5573 - --verbose env: - name: RCLONE_S3_PROVIDER @@ -153,6 +153,6 @@ spec: name: postgres-backups-775957147abfbc73-rclone-destination-config - name: RCLONE_CONFIG_DEST_FORCE_PATH_STYLE value: "true" - image: rclone/rclone:1.74.0@sha256:d2e0e88359d0b2e67cfcd2c43d5405185eb8adfc207079df27c42da82c5207bc + image: rclone/rclone:1.74.1@sha256:eb8788b795f0009324e9572b0e2acb9b56885327c2746b07d67a9d3b893a6602 imagePullPolicy: IfNotPresent name: sync diff --git a/clusters/cl01tl/manifests/cloudnative-pg/CronJob-cloudnative-pg-postgres-backups-remote-rclone.yaml b/clusters/cl01tl/manifests/cloudnative-pg/CronJob-cloudnative-pg-postgres-backups-remote-rclone.yaml index f34e31f47..7f2244d62 100644 --- a/clusters/cl01tl/manifests/cloudnative-pg/CronJob-cloudnative-pg-postgres-backups-remote-rclone.yaml +++ b/clusters/cl01tl/manifests/cloudnative-pg/CronJob-cloudnative-pg-postgres-backups-remote-rclone.yaml @@ -7,8 +7,8 @@ metadata: app.kubernetes.io/instance: cloudnative-pg app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: postgres-backups-remote-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-postgres-backups-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-postgres-backups-remote-0.11.0 namespace: cloudnative-pg spec: suspend: false @@ -46,7 +46,7 @@ spec: - --verbose - --rc - --rc-enable-metrics - - --metrics-addr=0.0.0.0:5572 + - --metrics-addr=0.0.0.0:5574 env: - name: RCLONE_CONFIG_DEST_TYPE value: s3 @@ -76,7 +76,7 @@ spec: name: postgres-backups-rclone-destination-config - name: RCLONE_CONFIG_DEST_FORCE_PATH_STYLE value: "true" - image: rclone/rclone:1.74.0@sha256:d2e0e88359d0b2e67cfcd2c43d5405185eb8adfc207079df27c42da82c5207bc + image: rclone/rclone:1.74.1@sha256:eb8788b795f0009324e9572b0e2acb9b56885327c2746b07d67a9d3b893a6602 imagePullPolicy: IfNotPresent name: prune - args: @@ -92,7 +92,7 @@ spec: - '**/walls/**' - --rc - --rc-enable-metrics - - --metrics-addr=0.0.0.0:5572 + - --metrics-addr=0.0.0.0:5573 - --verbose env: - name: RCLONE_S3_PROVIDER @@ -153,6 +153,6 @@ spec: name: postgres-backups-rclone-destination-config - name: RCLONE_CONFIG_DEST_FORCE_PATH_STYLE value: "true" - image: rclone/rclone:1.74.0@sha256:d2e0e88359d0b2e67cfcd2c43d5405185eb8adfc207079df27c42da82c5207bc + image: rclone/rclone:1.74.1@sha256:eb8788b795f0009324e9572b0e2acb9b56885327c2746b07d67a9d3b893a6602 imagePullPolicy: IfNotPresent name: sync diff --git a/clusters/cl01tl/manifests/cloudnative-pg/ExternalSecret-postgres-backups-775957147abfbc73-rclone-destination-config.yaml b/clusters/cl01tl/manifests/cloudnative-pg/ExternalSecret-postgres-backups-775957147abfbc73-rclone-destination-config.yaml index 7cce31b54..60225c287 100644 --- a/clusters/cl01tl/manifests/cloudnative-pg/ExternalSecret-postgres-backups-775957147abfbc73-rclone-destination-config.yaml +++ b/clusters/cl01tl/manifests/cloudnative-pg/ExternalSecret-postgres-backups-775957147abfbc73-rclone-destination-config.yaml @@ -4,10 +4,10 @@ metadata: name: postgres-backups-775957147abfbc73-rclone-destination-config namespace: cloudnative-pg labels: - helm.sh/chart: rclone-postgres-backups-external-0.9.1 + helm.sh/chart: rclone-postgres-backups-external-0.11.0 app.kubernetes.io/instance: cloudnative-pg app.kubernetes.io/part-of: cloudnative-pg - app.kubernetes.io/version: "0.9.1" + app.kubernetes.io/version: "0.11.0" app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: postgres-backups-775957147abfbc73-rclone-destination-config spec: diff --git a/clusters/cl01tl/manifests/cloudnative-pg/ExternalSecret-postgres-backups-rclone-destination-config.yaml b/clusters/cl01tl/manifests/cloudnative-pg/ExternalSecret-postgres-backups-rclone-destination-config.yaml index 1212b0cf7..e9dc957e8 100644 --- a/clusters/cl01tl/manifests/cloudnative-pg/ExternalSecret-postgres-backups-rclone-destination-config.yaml +++ b/clusters/cl01tl/manifests/cloudnative-pg/ExternalSecret-postgres-backups-rclone-destination-config.yaml @@ -4,10 +4,10 @@ metadata: name: postgres-backups-rclone-destination-config namespace: cloudnative-pg labels: - helm.sh/chart: rclone-postgres-backups-remote-0.9.1 + helm.sh/chart: rclone-postgres-backups-remote-0.11.0 app.kubernetes.io/instance: cloudnative-pg app.kubernetes.io/part-of: cloudnative-pg - app.kubernetes.io/version: "0.9.1" + app.kubernetes.io/version: "0.11.0" app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: postgres-backups-rclone-destination-config spec: diff --git a/clusters/cl01tl/manifests/cloudnative-pg/ExternalSecret-postgres-backups-rclone-source-config.yaml b/clusters/cl01tl/manifests/cloudnative-pg/ExternalSecret-postgres-backups-rclone-source-config.yaml index 0003c4c9b..14914b6a2 100644 --- a/clusters/cl01tl/manifests/cloudnative-pg/ExternalSecret-postgres-backups-rclone-source-config.yaml +++ b/clusters/cl01tl/manifests/cloudnative-pg/ExternalSecret-postgres-backups-rclone-source-config.yaml @@ -4,10 +4,10 @@ metadata: name: postgres-backups-rclone-source-config namespace: cloudnative-pg labels: - helm.sh/chart: rclone-postgres-backups-remote-0.9.1 + helm.sh/chart: rclone-postgres-backups-remote-0.11.0 app.kubernetes.io/instance: cloudnative-pg app.kubernetes.io/part-of: cloudnative-pg - app.kubernetes.io/version: "0.9.1" + app.kubernetes.io/version: "0.11.0" app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: postgres-backups-rclone-source-config spec: diff --git a/clusters/cl01tl/manifests/cloudnative-pg/PrometheusRule-postgres-backups-external-rclone.yaml b/clusters/cl01tl/manifests/cloudnative-pg/PrometheusRule-postgres-backups-external-rclone.yaml new file mode 100644 index 000000000..a43911174 --- /dev/null +++ b/clusters/cl01tl/manifests/cloudnative-pg/PrometheusRule-postgres-backups-external-rclone.yaml @@ -0,0 +1,30 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: postgres-backups-external-rclone + namespace: cloudnative-pg + labels: + helm.sh/chart: rclone-postgres-backups-external-0.11.0 + app.kubernetes.io/instance: cloudnative-pg + app.kubernetes.io/part-of: cloudnative-pg + app.kubernetes.io/version: "0.11.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: postgres-backups-external-rclone +spec: + groups: + - name: rclone.alerts + rules: + - alert: RclonePodFailed + expr: | + (kube_pod_container_status_last_terminated_exitcode > 0) + * on(pod, namespace) group_left(owner_name) + kube_pod_owner{owner_kind="Job", owner_name=~"rclone-.*"} + for: 1m + labels: + severity: critical + annotations: + summary: "Rclone Pod failed in {{ $labels.namespace }}" + description: | + A pod for the Rclone sync of s3 bucket 'postgres-backups' failed with exit code {{ $value }}. + Job: {{ $labels.owner_name }} + Namespace: {{ $labels.namespace }} diff --git a/clusters/cl01tl/manifests/cloudnative-pg/PrometheusRule-postgres-backups-remote-rclone.yaml b/clusters/cl01tl/manifests/cloudnative-pg/PrometheusRule-postgres-backups-remote-rclone.yaml new file mode 100644 index 000000000..75455fe7c --- /dev/null +++ b/clusters/cl01tl/manifests/cloudnative-pg/PrometheusRule-postgres-backups-remote-rclone.yaml @@ -0,0 +1,30 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: postgres-backups-remote-rclone + namespace: cloudnative-pg + labels: + helm.sh/chart: rclone-postgres-backups-remote-0.11.0 + app.kubernetes.io/instance: cloudnative-pg + app.kubernetes.io/part-of: cloudnative-pg + app.kubernetes.io/version: "0.11.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: postgres-backups-remote-rclone +spec: + groups: + - name: rclone.alerts + rules: + - alert: RclonePodFailed + expr: | + (kube_pod_container_status_last_terminated_exitcode > 0) + * on(pod, namespace) group_left(owner_name) + kube_pod_owner{owner_kind="Job", owner_name=~"rclone-.*"} + for: 1m + labels: + severity: critical + annotations: + summary: "Rclone Pod failed in {{ $labels.namespace }}" + description: | + A pod for the Rclone sync of s3 bucket 'postgres-backups' failed with exit code {{ $value }}. + Job: {{ $labels.owner_name }} + Namespace: {{ $labels.namespace }} diff --git a/clusters/cl01tl/manifests/cloudnative-pg/Service-cloudnative-pg-postgres-backups-external-rclone.yaml b/clusters/cl01tl/manifests/cloudnative-pg/Service-cloudnative-pg-postgres-backups-external-rclone.yaml index 0804e9da4..11eb50316 100644 --- a/clusters/cl01tl/manifests/cloudnative-pg/Service-cloudnative-pg-postgres-backups-external-rclone.yaml +++ b/clusters/cl01tl/manifests/cloudnative-pg/Service-cloudnative-pg-postgres-backups-external-rclone.yaml @@ -7,16 +7,24 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: postgres-backups-external-rclone app.kubernetes.io/service: cloudnative-pg-postgres-backups-external-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-postgres-backups-external-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-postgres-backups-external-0.11.0 namespace: cloudnative-pg spec: type: ClusterIP ports: + - port: 5574 + targetPort: 5574 + protocol: TCP + name: metrics-prune + - port: 5573 + targetPort: 5573 + protocol: TCP + name: metrics-sync - port: 5572 targetPort: 5572 protocol: TCP - name: metrics + name: rc selector: app.kubernetes.io/controller: main app.kubernetes.io/instance: cloudnative-pg diff --git a/clusters/cl01tl/manifests/cloudnative-pg/Service-cloudnative-pg-postgres-backups-remote-rclone.yaml b/clusters/cl01tl/manifests/cloudnative-pg/Service-cloudnative-pg-postgres-backups-remote-rclone.yaml index 9fb44e940..ce096b616 100644 --- a/clusters/cl01tl/manifests/cloudnative-pg/Service-cloudnative-pg-postgres-backups-remote-rclone.yaml +++ b/clusters/cl01tl/manifests/cloudnative-pg/Service-cloudnative-pg-postgres-backups-remote-rclone.yaml @@ -7,16 +7,24 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: postgres-backups-remote-rclone app.kubernetes.io/service: cloudnative-pg-postgres-backups-remote-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-postgres-backups-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-postgres-backups-remote-0.11.0 namespace: cloudnative-pg spec: type: ClusterIP ports: + - port: 5574 + targetPort: 5574 + protocol: TCP + name: metrics-prune + - port: 5573 + targetPort: 5573 + protocol: TCP + name: metrics-sync - port: 5572 targetPort: 5572 protocol: TCP - name: metrics + name: rc selector: app.kubernetes.io/controller: main app.kubernetes.io/instance: cloudnative-pg diff --git a/clusters/cl01tl/manifests/cloudnative-pg/ServiceAccount-cloudnative-pg-postgres-backups-external-rclone.yaml b/clusters/cl01tl/manifests/cloudnative-pg/ServiceAccount-cloudnative-pg-postgres-backups-external-rclone.yaml index e89d4274d..81e90ed5b 100644 --- a/clusters/cl01tl/manifests/cloudnative-pg/ServiceAccount-cloudnative-pg-postgres-backups-external-rclone.yaml +++ b/clusters/cl01tl/manifests/cloudnative-pg/ServiceAccount-cloudnative-pg-postgres-backups-external-rclone.yaml @@ -6,6 +6,6 @@ metadata: app.kubernetes.io/instance: cloudnative-pg app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: postgres-backups-external-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-postgres-backups-external-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-postgres-backups-external-0.11.0 namespace: cloudnative-pg diff --git a/clusters/cl01tl/manifests/cloudnative-pg/ServiceAccount-cloudnative-pg-postgres-backups-remote-rclone.yaml b/clusters/cl01tl/manifests/cloudnative-pg/ServiceAccount-cloudnative-pg-postgres-backups-remote-rclone.yaml index 7f56b546e..e08b2de9d 100644 --- a/clusters/cl01tl/manifests/cloudnative-pg/ServiceAccount-cloudnative-pg-postgres-backups-remote-rclone.yaml +++ b/clusters/cl01tl/manifests/cloudnative-pg/ServiceAccount-cloudnative-pg-postgres-backups-remote-rclone.yaml @@ -6,6 +6,6 @@ metadata: app.kubernetes.io/instance: cloudnative-pg app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: postgres-backups-remote-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-postgres-backups-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-postgres-backups-remote-0.11.0 namespace: cloudnative-pg diff --git a/clusters/cl01tl/manifests/cloudnative-pg/ServiceMonitor-cloudnative-pg-postgres-backups-external-rclone.yaml b/clusters/cl01tl/manifests/cloudnative-pg/ServiceMonitor-cloudnative-pg-postgres-backups-external-rclone.yaml index 86ec9035a..a3ebd1640 100644 --- a/clusters/cl01tl/manifests/cloudnative-pg/ServiceMonitor-cloudnative-pg-postgres-backups-external-rclone.yaml +++ b/clusters/cl01tl/manifests/cloudnative-pg/ServiceMonitor-cloudnative-pg-postgres-backups-external-rclone.yaml @@ -6,8 +6,8 @@ metadata: app.kubernetes.io/instance: cloudnative-pg app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: postgres-backups-external-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-postgres-backups-external-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-postgres-backups-external-0.11.0 namespace: cloudnative-pg spec: jobLabel: app.kubernetes.io/name @@ -19,7 +19,11 @@ spec: app.kubernetes.io/instance: cloudnative-pg app.kubernetes.io/name: postgres-backups-external-rclone endpoints: - - interval: 30s + - interval: 10s path: /metrics - port: metrics - scrapeTimeout: 10s + port: metrics-sync + scrapeTimeout: 5s + - interval: 10s + path: /metrics + port: metrics-prune + scrapeTimeout: 5s diff --git a/clusters/cl01tl/manifests/cloudnative-pg/ServiceMonitor-cloudnative-pg-postgres-backups-remote-rclone.yaml b/clusters/cl01tl/manifests/cloudnative-pg/ServiceMonitor-cloudnative-pg-postgres-backups-remote-rclone.yaml index 79844efd4..3ff4c459e 100644 --- a/clusters/cl01tl/manifests/cloudnative-pg/ServiceMonitor-cloudnative-pg-postgres-backups-remote-rclone.yaml +++ b/clusters/cl01tl/manifests/cloudnative-pg/ServiceMonitor-cloudnative-pg-postgres-backups-remote-rclone.yaml @@ -6,8 +6,8 @@ metadata: app.kubernetes.io/instance: cloudnative-pg app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: postgres-backups-remote-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-postgres-backups-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-postgres-backups-remote-0.11.0 namespace: cloudnative-pg spec: jobLabel: app.kubernetes.io/name @@ -19,7 +19,11 @@ spec: app.kubernetes.io/instance: cloudnative-pg app.kubernetes.io/name: postgres-backups-remote-rclone endpoints: - - interval: 30s + - interval: 10s path: /metrics - port: metrics - scrapeTimeout: 10s + port: metrics-sync + scrapeTimeout: 5s + - interval: 10s + path: /metrics + port: metrics-prune + scrapeTimeout: 5s diff --git a/clusters/cl01tl/manifests/directus/CronJob-directus-directus-assets-external-rclone.yaml b/clusters/cl01tl/manifests/directus/CronJob-directus-directus-assets-external-rclone.yaml index a3efdef69..e543d8a8f 100644 --- a/clusters/cl01tl/manifests/directus/CronJob-directus-directus-assets-external-rclone.yaml +++ b/clusters/cl01tl/manifests/directus/CronJob-directus-directus-assets-external-rclone.yaml @@ -7,8 +7,8 @@ metadata: app.kubernetes.io/instance: directus app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: directus-assets-external-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-directus-assets-external-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-directus-assets-external-0.11.0 namespace: directus spec: suspend: false @@ -45,7 +45,7 @@ spec: - --s3-no-check-bucket - --rc - --rc-enable-metrics - - --metrics-addr=0.0.0.0:5572 + - --metrics-addr=0.0.0.0:5573 - --verbose env: - name: RCLONE_S3_PROVIDER @@ -106,6 +106,6 @@ spec: name: directus-assets-37363a16b71dc59b-rclone-destination-config - name: RCLONE_CONFIG_DEST_FORCE_PATH_STYLE value: "true" - image: rclone/rclone:1.74.0@sha256:d2e0e88359d0b2e67cfcd2c43d5405185eb8adfc207079df27c42da82c5207bc + image: rclone/rclone:1.74.1@sha256:eb8788b795f0009324e9572b0e2acb9b56885327c2746b07d67a9d3b893a6602 imagePullPolicy: IfNotPresent name: sync diff --git a/clusters/cl01tl/manifests/directus/CronJob-directus-directus-assets-remote-rclone.yaml b/clusters/cl01tl/manifests/directus/CronJob-directus-directus-assets-remote-rclone.yaml index 1bd3ef600..1954f5694 100644 --- a/clusters/cl01tl/manifests/directus/CronJob-directus-directus-assets-remote-rclone.yaml +++ b/clusters/cl01tl/manifests/directus/CronJob-directus-directus-assets-remote-rclone.yaml @@ -7,8 +7,8 @@ metadata: app.kubernetes.io/instance: directus app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: directus-assets-remote-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-directus-assets-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-directus-assets-remote-0.11.0 namespace: directus spec: suspend: false @@ -45,7 +45,7 @@ spec: - --s3-no-check-bucket - --rc - --rc-enable-metrics - - --metrics-addr=0.0.0.0:5572 + - --metrics-addr=0.0.0.0:5573 - --verbose env: - name: RCLONE_S3_PROVIDER @@ -106,6 +106,6 @@ spec: name: directus-assets-rclone-destination-config - name: RCLONE_CONFIG_DEST_FORCE_PATH_STYLE value: "true" - image: rclone/rclone:1.74.0@sha256:d2e0e88359d0b2e67cfcd2c43d5405185eb8adfc207079df27c42da82c5207bc + image: rclone/rclone:1.74.1@sha256:eb8788b795f0009324e9572b0e2acb9b56885327c2746b07d67a9d3b893a6602 imagePullPolicy: IfNotPresent name: sync diff --git a/clusters/cl01tl/manifests/directus/ExternalSecret-directus-assets-37363a16b71dc59b-rclone-destination-config.yaml b/clusters/cl01tl/manifests/directus/ExternalSecret-directus-assets-37363a16b71dc59b-rclone-destination-config.yaml index 8a241fca1..55c5ac56f 100644 --- a/clusters/cl01tl/manifests/directus/ExternalSecret-directus-assets-37363a16b71dc59b-rclone-destination-config.yaml +++ b/clusters/cl01tl/manifests/directus/ExternalSecret-directus-assets-37363a16b71dc59b-rclone-destination-config.yaml @@ -4,10 +4,10 @@ metadata: name: directus-assets-37363a16b71dc59b-rclone-destination-config namespace: directus labels: - helm.sh/chart: rclone-directus-assets-external-0.9.1 + helm.sh/chart: rclone-directus-assets-external-0.11.0 app.kubernetes.io/instance: directus app.kubernetes.io/part-of: directus - app.kubernetes.io/version: "0.9.1" + app.kubernetes.io/version: "0.11.0" app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: directus-assets-37363a16b71dc59b-rclone-destination-config spec: diff --git a/clusters/cl01tl/manifests/directus/ExternalSecret-directus-assets-rclone-destination-config.yaml b/clusters/cl01tl/manifests/directus/ExternalSecret-directus-assets-rclone-destination-config.yaml index 0e826d583..9f6f30be4 100644 --- a/clusters/cl01tl/manifests/directus/ExternalSecret-directus-assets-rclone-destination-config.yaml +++ b/clusters/cl01tl/manifests/directus/ExternalSecret-directus-assets-rclone-destination-config.yaml @@ -4,10 +4,10 @@ metadata: name: directus-assets-rclone-destination-config namespace: directus labels: - helm.sh/chart: rclone-directus-assets-remote-0.9.1 + helm.sh/chart: rclone-directus-assets-remote-0.11.0 app.kubernetes.io/instance: directus app.kubernetes.io/part-of: directus - app.kubernetes.io/version: "0.9.1" + app.kubernetes.io/version: "0.11.0" app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: directus-assets-rclone-destination-config spec: diff --git a/clusters/cl01tl/manifests/directus/ExternalSecret-directus-assets-rclone-source-config.yaml b/clusters/cl01tl/manifests/directus/ExternalSecret-directus-assets-rclone-source-config.yaml index 5dfc28ea3..c8839e70e 100644 --- a/clusters/cl01tl/manifests/directus/ExternalSecret-directus-assets-rclone-source-config.yaml +++ b/clusters/cl01tl/manifests/directus/ExternalSecret-directus-assets-rclone-source-config.yaml @@ -4,10 +4,10 @@ metadata: name: directus-assets-rclone-source-config namespace: directus labels: - helm.sh/chart: rclone-directus-assets-remote-0.9.1 + helm.sh/chart: rclone-directus-assets-remote-0.11.0 app.kubernetes.io/instance: directus app.kubernetes.io/part-of: directus - app.kubernetes.io/version: "0.9.1" + app.kubernetes.io/version: "0.11.0" app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: directus-assets-rclone-source-config spec: diff --git a/clusters/cl01tl/manifests/directus/PrometheusRule-directus-assets-external-rclone.yaml b/clusters/cl01tl/manifests/directus/PrometheusRule-directus-assets-external-rclone.yaml new file mode 100644 index 000000000..5f55078ab --- /dev/null +++ b/clusters/cl01tl/manifests/directus/PrometheusRule-directus-assets-external-rclone.yaml @@ -0,0 +1,30 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: directus-assets-external-rclone + namespace: directus + labels: + helm.sh/chart: rclone-directus-assets-external-0.11.0 + app.kubernetes.io/instance: directus + app.kubernetes.io/part-of: directus + app.kubernetes.io/version: "0.11.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: directus-assets-external-rclone +spec: + groups: + - name: rclone.alerts + rules: + - alert: RclonePodFailed + expr: | + (kube_pod_container_status_last_terminated_exitcode > 0) + * on(pod, namespace) group_left(owner_name) + kube_pod_owner{owner_kind="Job", owner_name=~"rclone-.*"} + for: 1m + labels: + severity: critical + annotations: + summary: "Rclone Pod failed in {{ $labels.namespace }}" + description: | + A pod for the Rclone sync of s3 bucket 'directus-assets' failed with exit code {{ $value }}. + Job: {{ $labels.owner_name }} + Namespace: {{ $labels.namespace }} diff --git a/clusters/cl01tl/manifests/directus/PrometheusRule-directus-assets-remote-rclone.yaml b/clusters/cl01tl/manifests/directus/PrometheusRule-directus-assets-remote-rclone.yaml new file mode 100644 index 000000000..a95064d56 --- /dev/null +++ b/clusters/cl01tl/manifests/directus/PrometheusRule-directus-assets-remote-rclone.yaml @@ -0,0 +1,30 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: directus-assets-remote-rclone + namespace: directus + labels: + helm.sh/chart: rclone-directus-assets-remote-0.11.0 + app.kubernetes.io/instance: directus + app.kubernetes.io/part-of: directus + app.kubernetes.io/version: "0.11.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: directus-assets-remote-rclone +spec: + groups: + - name: rclone.alerts + rules: + - alert: RclonePodFailed + expr: | + (kube_pod_container_status_last_terminated_exitcode > 0) + * on(pod, namespace) group_left(owner_name) + kube_pod_owner{owner_kind="Job", owner_name=~"rclone-.*"} + for: 1m + labels: + severity: critical + annotations: + summary: "Rclone Pod failed in {{ $labels.namespace }}" + description: | + A pod for the Rclone sync of s3 bucket 'directus-assets' failed with exit code {{ $value }}. + Job: {{ $labels.owner_name }} + Namespace: {{ $labels.namespace }} diff --git a/clusters/cl01tl/manifests/directus/Service-directus-directus-assets-external-rclone.yaml b/clusters/cl01tl/manifests/directus/Service-directus-directus-assets-external-rclone.yaml index 791c83ea6..68bcd6993 100644 --- a/clusters/cl01tl/manifests/directus/Service-directus-directus-assets-external-rclone.yaml +++ b/clusters/cl01tl/manifests/directus/Service-directus-directus-assets-external-rclone.yaml @@ -7,16 +7,24 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: directus-assets-external-rclone app.kubernetes.io/service: directus-directus-assets-external-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-directus-assets-external-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-directus-assets-external-0.11.0 namespace: directus spec: type: ClusterIP ports: + - port: 5574 + targetPort: 5574 + protocol: TCP + name: metrics-prune + - port: 5573 + targetPort: 5573 + protocol: TCP + name: metrics-sync - port: 5572 targetPort: 5572 protocol: TCP - name: metrics + name: rc selector: app.kubernetes.io/controller: main app.kubernetes.io/instance: directus diff --git a/clusters/cl01tl/manifests/directus/Service-directus-directus-assets-remote-rclone.yaml b/clusters/cl01tl/manifests/directus/Service-directus-directus-assets-remote-rclone.yaml index 0712ca5ee..f0293aee7 100644 --- a/clusters/cl01tl/manifests/directus/Service-directus-directus-assets-remote-rclone.yaml +++ b/clusters/cl01tl/manifests/directus/Service-directus-directus-assets-remote-rclone.yaml @@ -7,16 +7,24 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: directus-assets-remote-rclone app.kubernetes.io/service: directus-directus-assets-remote-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-directus-assets-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-directus-assets-remote-0.11.0 namespace: directus spec: type: ClusterIP ports: + - port: 5574 + targetPort: 5574 + protocol: TCP + name: metrics-prune + - port: 5573 + targetPort: 5573 + protocol: TCP + name: metrics-sync - port: 5572 targetPort: 5572 protocol: TCP - name: metrics + name: rc selector: app.kubernetes.io/controller: main app.kubernetes.io/instance: directus diff --git a/clusters/cl01tl/manifests/directus/ServiceAccount-directus-directus-assets-external-rclone.yaml b/clusters/cl01tl/manifests/directus/ServiceAccount-directus-directus-assets-external-rclone.yaml index de477d271..051633b7a 100644 --- a/clusters/cl01tl/manifests/directus/ServiceAccount-directus-directus-assets-external-rclone.yaml +++ b/clusters/cl01tl/manifests/directus/ServiceAccount-directus-directus-assets-external-rclone.yaml @@ -6,6 +6,6 @@ metadata: app.kubernetes.io/instance: directus app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: directus-assets-external-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-directus-assets-external-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-directus-assets-external-0.11.0 namespace: directus diff --git a/clusters/cl01tl/manifests/directus/ServiceAccount-directus-directus-assets-remote-rclone.yaml b/clusters/cl01tl/manifests/directus/ServiceAccount-directus-directus-assets-remote-rclone.yaml index fab27da24..dd570e987 100644 --- a/clusters/cl01tl/manifests/directus/ServiceAccount-directus-directus-assets-remote-rclone.yaml +++ b/clusters/cl01tl/manifests/directus/ServiceAccount-directus-directus-assets-remote-rclone.yaml @@ -6,6 +6,6 @@ metadata: app.kubernetes.io/instance: directus app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: directus-assets-remote-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-directus-assets-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-directus-assets-remote-0.11.0 namespace: directus diff --git a/clusters/cl01tl/manifests/directus/ServiceMonitor-directus-directus-assets-external-rclone.yaml b/clusters/cl01tl/manifests/directus/ServiceMonitor-directus-directus-assets-external-rclone.yaml index fd2fa8835..95011169d 100644 --- a/clusters/cl01tl/manifests/directus/ServiceMonitor-directus-directus-assets-external-rclone.yaml +++ b/clusters/cl01tl/manifests/directus/ServiceMonitor-directus-directus-assets-external-rclone.yaml @@ -6,8 +6,8 @@ metadata: app.kubernetes.io/instance: directus app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: directus-assets-external-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-directus-assets-external-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-directus-assets-external-0.11.0 namespace: directus spec: jobLabel: app.kubernetes.io/name @@ -19,7 +19,11 @@ spec: app.kubernetes.io/instance: directus app.kubernetes.io/name: directus-assets-external-rclone endpoints: - - interval: 30s + - interval: 10s path: /metrics - port: metrics - scrapeTimeout: 10s + port: metrics-sync + scrapeTimeout: 5s + - interval: 10s + path: /metrics + port: metrics-prune + scrapeTimeout: 5s diff --git a/clusters/cl01tl/manifests/directus/ServiceMonitor-directus-directus-assets-remote-rclone.yaml b/clusters/cl01tl/manifests/directus/ServiceMonitor-directus-directus-assets-remote-rclone.yaml index 3924230c1..963c98562 100644 --- a/clusters/cl01tl/manifests/directus/ServiceMonitor-directus-directus-assets-remote-rclone.yaml +++ b/clusters/cl01tl/manifests/directus/ServiceMonitor-directus-directus-assets-remote-rclone.yaml @@ -6,8 +6,8 @@ metadata: app.kubernetes.io/instance: directus app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: directus-assets-remote-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-directus-assets-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-directus-assets-remote-0.11.0 namespace: directus spec: jobLabel: app.kubernetes.io/name @@ -19,7 +19,11 @@ spec: app.kubernetes.io/instance: directus app.kubernetes.io/name: directus-assets-remote-rclone endpoints: - - interval: 30s + - interval: 10s path: /metrics - port: metrics - scrapeTimeout: 10s + port: metrics-sync + scrapeTimeout: 5s + - interval: 10s + path: /metrics + port: metrics-prune + scrapeTimeout: 5s diff --git a/clusters/cl01tl/manifests/karakeep/CronJob-karakeep-karakeep-assets-external-rclone.yaml b/clusters/cl01tl/manifests/karakeep/CronJob-karakeep-karakeep-assets-external-rclone.yaml index c2ffc5164..3e0b7a960 100644 --- a/clusters/cl01tl/manifests/karakeep/CronJob-karakeep-karakeep-assets-external-rclone.yaml +++ b/clusters/cl01tl/manifests/karakeep/CronJob-karakeep-karakeep-assets-external-rclone.yaml @@ -7,8 +7,8 @@ metadata: app.kubernetes.io/instance: karakeep app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: karakeep-assets-external-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-karakeep-assets-external-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-karakeep-assets-external-0.11.0 namespace: karakeep spec: suspend: false @@ -45,7 +45,7 @@ spec: - --s3-no-check-bucket - --rc - --rc-enable-metrics - - --metrics-addr=0.0.0.0:5572 + - --metrics-addr=0.0.0.0:5573 - --verbose env: - name: RCLONE_S3_PROVIDER @@ -106,6 +106,6 @@ spec: name: karakeep-assets-bcb0bc04dac3e3fd-rclone-destination-config - name: RCLONE_CONFIG_DEST_FORCE_PATH_STYLE value: "true" - image: rclone/rclone:1.74.0@sha256:d2e0e88359d0b2e67cfcd2c43d5405185eb8adfc207079df27c42da82c5207bc + image: rclone/rclone:1.74.1@sha256:eb8788b795f0009324e9572b0e2acb9b56885327c2746b07d67a9d3b893a6602 imagePullPolicy: IfNotPresent name: sync diff --git a/clusters/cl01tl/manifests/karakeep/CronJob-karakeep-karakeep-assets-rclone.yaml b/clusters/cl01tl/manifests/karakeep/CronJob-karakeep-karakeep-assets-rclone.yaml index fb51be8ac..18c8454e0 100644 --- a/clusters/cl01tl/manifests/karakeep/CronJob-karakeep-karakeep-assets-rclone.yaml +++ b/clusters/cl01tl/manifests/karakeep/CronJob-karakeep-karakeep-assets-rclone.yaml @@ -7,8 +7,8 @@ metadata: app.kubernetes.io/instance: karakeep app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: karakeep-assets-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-karakeep-assets-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-karakeep-assets-remote-0.11.0 namespace: karakeep spec: suspend: false @@ -45,7 +45,7 @@ spec: - --s3-no-check-bucket - --rc - --rc-enable-metrics - - --metrics-addr=0.0.0.0:5572 + - --metrics-addr=0.0.0.0:5573 - --verbose env: - name: RCLONE_S3_PROVIDER @@ -106,6 +106,6 @@ spec: name: karakeep-assets-rclone-destination-config - name: RCLONE_CONFIG_DEST_FORCE_PATH_STYLE value: "true" - image: rclone/rclone:1.74.0@sha256:d2e0e88359d0b2e67cfcd2c43d5405185eb8adfc207079df27c42da82c5207bc + image: rclone/rclone:1.74.1@sha256:eb8788b795f0009324e9572b0e2acb9b56885327c2746b07d67a9d3b893a6602 imagePullPolicy: IfNotPresent name: sync diff --git a/clusters/cl01tl/manifests/karakeep/ExternalSecret-karakeep-assets-bcb0bc04dac3e3fd-rclone-destination-config.yaml b/clusters/cl01tl/manifests/karakeep/ExternalSecret-karakeep-assets-bcb0bc04dac3e3fd-rclone-destination-config.yaml index 0dbb630cd..83eca7298 100644 --- a/clusters/cl01tl/manifests/karakeep/ExternalSecret-karakeep-assets-bcb0bc04dac3e3fd-rclone-destination-config.yaml +++ b/clusters/cl01tl/manifests/karakeep/ExternalSecret-karakeep-assets-bcb0bc04dac3e3fd-rclone-destination-config.yaml @@ -4,10 +4,10 @@ metadata: name: karakeep-assets-bcb0bc04dac3e3fd-rclone-destination-config namespace: karakeep labels: - helm.sh/chart: rclone-karakeep-assets-external-0.9.1 + helm.sh/chart: rclone-karakeep-assets-external-0.11.0 app.kubernetes.io/instance: karakeep app.kubernetes.io/part-of: karakeep - app.kubernetes.io/version: "0.9.1" + app.kubernetes.io/version: "0.11.0" app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: karakeep-assets-bcb0bc04dac3e3fd-rclone-destination-config spec: diff --git a/clusters/cl01tl/manifests/karakeep/ExternalSecret-karakeep-assets-rclone-destination-config.yaml b/clusters/cl01tl/manifests/karakeep/ExternalSecret-karakeep-assets-rclone-destination-config.yaml index d994eee66..88544a786 100644 --- a/clusters/cl01tl/manifests/karakeep/ExternalSecret-karakeep-assets-rclone-destination-config.yaml +++ b/clusters/cl01tl/manifests/karakeep/ExternalSecret-karakeep-assets-rclone-destination-config.yaml @@ -4,10 +4,10 @@ metadata: name: karakeep-assets-rclone-destination-config namespace: karakeep labels: - helm.sh/chart: rclone-karakeep-assets-remote-0.9.1 + helm.sh/chart: rclone-karakeep-assets-remote-0.11.0 app.kubernetes.io/instance: karakeep app.kubernetes.io/part-of: karakeep - app.kubernetes.io/version: "0.9.1" + app.kubernetes.io/version: "0.11.0" app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: karakeep-assets-rclone-destination-config spec: diff --git a/clusters/cl01tl/manifests/karakeep/ExternalSecret-karakeep-assets-rclone-source-config.yaml b/clusters/cl01tl/manifests/karakeep/ExternalSecret-karakeep-assets-rclone-source-config.yaml index c7d1263fb..d23205d1e 100644 --- a/clusters/cl01tl/manifests/karakeep/ExternalSecret-karakeep-assets-rclone-source-config.yaml +++ b/clusters/cl01tl/manifests/karakeep/ExternalSecret-karakeep-assets-rclone-source-config.yaml @@ -4,10 +4,10 @@ metadata: name: karakeep-assets-rclone-source-config namespace: karakeep labels: - helm.sh/chart: rclone-karakeep-assets-remote-0.9.1 + helm.sh/chart: rclone-karakeep-assets-remote-0.11.0 app.kubernetes.io/instance: karakeep app.kubernetes.io/part-of: karakeep - app.kubernetes.io/version: "0.9.1" + app.kubernetes.io/version: "0.11.0" app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: karakeep-assets-rclone-source-config spec: diff --git a/clusters/cl01tl/manifests/karakeep/PrometheusRule-karakeep-assets-external-rclone.yaml b/clusters/cl01tl/manifests/karakeep/PrometheusRule-karakeep-assets-external-rclone.yaml new file mode 100644 index 000000000..b75fabaa5 --- /dev/null +++ b/clusters/cl01tl/manifests/karakeep/PrometheusRule-karakeep-assets-external-rclone.yaml @@ -0,0 +1,30 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: karakeep-assets-external-rclone + namespace: karakeep + labels: + helm.sh/chart: rclone-karakeep-assets-external-0.11.0 + app.kubernetes.io/instance: karakeep + app.kubernetes.io/part-of: karakeep + app.kubernetes.io/version: "0.11.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: karakeep-assets-external-rclone +spec: + groups: + - name: rclone.alerts + rules: + - alert: RclonePodFailed + expr: | + (kube_pod_container_status_last_terminated_exitcode > 0) + * on(pod, namespace) group_left(owner_name) + kube_pod_owner{owner_kind="Job", owner_name=~"rclone-.*"} + for: 1m + labels: + severity: critical + annotations: + summary: "Rclone Pod failed in {{ $labels.namespace }}" + description: | + A pod for the Rclone sync of s3 bucket 'karakeep-assets' failed with exit code {{ $value }}. + Job: {{ $labels.owner_name }} + Namespace: {{ $labels.namespace }} diff --git a/clusters/cl01tl/manifests/karakeep/PrometheusRule-karakeep-assets-rclone.yaml b/clusters/cl01tl/manifests/karakeep/PrometheusRule-karakeep-assets-rclone.yaml new file mode 100644 index 000000000..48c053c1a --- /dev/null +++ b/clusters/cl01tl/manifests/karakeep/PrometheusRule-karakeep-assets-rclone.yaml @@ -0,0 +1,30 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: karakeep-assets-rclone + namespace: karakeep + labels: + helm.sh/chart: rclone-karakeep-assets-remote-0.11.0 + app.kubernetes.io/instance: karakeep + app.kubernetes.io/part-of: karakeep + app.kubernetes.io/version: "0.11.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: karakeep-assets-rclone +spec: + groups: + - name: rclone.alerts + rules: + - alert: RclonePodFailed + expr: | + (kube_pod_container_status_last_terminated_exitcode > 0) + * on(pod, namespace) group_left(owner_name) + kube_pod_owner{owner_kind="Job", owner_name=~"rclone-.*"} + for: 1m + labels: + severity: critical + annotations: + summary: "Rclone Pod failed in {{ $labels.namespace }}" + description: | + A pod for the Rclone sync of s3 bucket 'karakeep-assets' failed with exit code {{ $value }}. + Job: {{ $labels.owner_name }} + Namespace: {{ $labels.namespace }} diff --git a/clusters/cl01tl/manifests/karakeep/Service-karakeep-karakeep-assets-external-rclone.yaml b/clusters/cl01tl/manifests/karakeep/Service-karakeep-karakeep-assets-external-rclone.yaml index 55d334f9d..dbe1c25a4 100644 --- a/clusters/cl01tl/manifests/karakeep/Service-karakeep-karakeep-assets-external-rclone.yaml +++ b/clusters/cl01tl/manifests/karakeep/Service-karakeep-karakeep-assets-external-rclone.yaml @@ -7,16 +7,24 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: karakeep-assets-external-rclone app.kubernetes.io/service: karakeep-karakeep-assets-external-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-karakeep-assets-external-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-karakeep-assets-external-0.11.0 namespace: karakeep spec: type: ClusterIP ports: + - port: 5574 + targetPort: 5574 + protocol: TCP + name: metrics-prune + - port: 5573 + targetPort: 5573 + protocol: TCP + name: metrics-sync - port: 5572 targetPort: 5572 protocol: TCP - name: metrics + name: rc selector: app.kubernetes.io/controller: main app.kubernetes.io/instance: karakeep diff --git a/clusters/cl01tl/manifests/karakeep/Service-karakeep-karakeep-assets-rclone.yaml b/clusters/cl01tl/manifests/karakeep/Service-karakeep-karakeep-assets-rclone.yaml index 85e14cdb5..9afdd0d94 100644 --- a/clusters/cl01tl/manifests/karakeep/Service-karakeep-karakeep-assets-rclone.yaml +++ b/clusters/cl01tl/manifests/karakeep/Service-karakeep-karakeep-assets-rclone.yaml @@ -7,16 +7,24 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: karakeep-assets-rclone app.kubernetes.io/service: karakeep-karakeep-assets-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-karakeep-assets-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-karakeep-assets-remote-0.11.0 namespace: karakeep spec: type: ClusterIP ports: + - port: 5574 + targetPort: 5574 + protocol: TCP + name: metrics-prune + - port: 5573 + targetPort: 5573 + protocol: TCP + name: metrics-sync - port: 5572 targetPort: 5572 protocol: TCP - name: metrics + name: rc selector: app.kubernetes.io/controller: main app.kubernetes.io/instance: karakeep diff --git a/clusters/cl01tl/manifests/karakeep/ServiceAccount-karakeep-karakeep-assets-external-rclone.yaml b/clusters/cl01tl/manifests/karakeep/ServiceAccount-karakeep-karakeep-assets-external-rclone.yaml index 5da33ca0d..2c86d2048 100644 --- a/clusters/cl01tl/manifests/karakeep/ServiceAccount-karakeep-karakeep-assets-external-rclone.yaml +++ b/clusters/cl01tl/manifests/karakeep/ServiceAccount-karakeep-karakeep-assets-external-rclone.yaml @@ -6,6 +6,6 @@ metadata: app.kubernetes.io/instance: karakeep app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: karakeep-assets-external-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-karakeep-assets-external-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-karakeep-assets-external-0.11.0 namespace: karakeep diff --git a/clusters/cl01tl/manifests/karakeep/ServiceAccount-karakeep-karakeep-assets-rclone.yaml b/clusters/cl01tl/manifests/karakeep/ServiceAccount-karakeep-karakeep-assets-rclone.yaml index 966ea73e5..d628e7053 100644 --- a/clusters/cl01tl/manifests/karakeep/ServiceAccount-karakeep-karakeep-assets-rclone.yaml +++ b/clusters/cl01tl/manifests/karakeep/ServiceAccount-karakeep-karakeep-assets-rclone.yaml @@ -6,6 +6,6 @@ metadata: app.kubernetes.io/instance: karakeep app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: karakeep-assets-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-karakeep-assets-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-karakeep-assets-remote-0.11.0 namespace: karakeep diff --git a/clusters/cl01tl/manifests/karakeep/ServiceMonitor-karakeep-karakeep-assets-external-rclone.yaml b/clusters/cl01tl/manifests/karakeep/ServiceMonitor-karakeep-karakeep-assets-external-rclone.yaml index 5833185cd..8544e907c 100644 --- a/clusters/cl01tl/manifests/karakeep/ServiceMonitor-karakeep-karakeep-assets-external-rclone.yaml +++ b/clusters/cl01tl/manifests/karakeep/ServiceMonitor-karakeep-karakeep-assets-external-rclone.yaml @@ -6,8 +6,8 @@ metadata: app.kubernetes.io/instance: karakeep app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: karakeep-assets-external-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-karakeep-assets-external-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-karakeep-assets-external-0.11.0 namespace: karakeep spec: jobLabel: app.kubernetes.io/name @@ -19,7 +19,11 @@ spec: app.kubernetes.io/instance: karakeep app.kubernetes.io/name: karakeep-assets-external-rclone endpoints: - - interval: 30s + - interval: 10s path: /metrics - port: metrics - scrapeTimeout: 10s + port: metrics-sync + scrapeTimeout: 5s + - interval: 10s + path: /metrics + port: metrics-prune + scrapeTimeout: 5s diff --git a/clusters/cl01tl/manifests/karakeep/ServiceMonitor-karakeep-karakeep-assets-rclone.yaml b/clusters/cl01tl/manifests/karakeep/ServiceMonitor-karakeep-karakeep-assets-rclone.yaml index 78e22c575..f938ec754 100644 --- a/clusters/cl01tl/manifests/karakeep/ServiceMonitor-karakeep-karakeep-assets-rclone.yaml +++ b/clusters/cl01tl/manifests/karakeep/ServiceMonitor-karakeep-karakeep-assets-rclone.yaml @@ -6,8 +6,8 @@ metadata: app.kubernetes.io/instance: karakeep app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: karakeep-assets-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-karakeep-assets-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-karakeep-assets-remote-0.11.0 namespace: karakeep spec: jobLabel: app.kubernetes.io/name @@ -19,7 +19,11 @@ spec: app.kubernetes.io/instance: karakeep app.kubernetes.io/name: karakeep-assets-rclone endpoints: - - interval: 30s + - interval: 10s path: /metrics - port: metrics - scrapeTimeout: 10s + port: metrics-sync + scrapeTimeout: 5s + - interval: 10s + path: /metrics + port: metrics-prune + scrapeTimeout: 5s diff --git a/clusters/cl01tl/manifests/ntfy/CronJob-ntfy-ntfy-attachments-rclone.yaml b/clusters/cl01tl/manifests/ntfy/CronJob-ntfy-ntfy-attachments-rclone.yaml index fd46a3dd2..bec8fc7ea 100644 --- a/clusters/cl01tl/manifests/ntfy/CronJob-ntfy-ntfy-attachments-rclone.yaml +++ b/clusters/cl01tl/manifests/ntfy/CronJob-ntfy-ntfy-attachments-rclone.yaml @@ -7,8 +7,8 @@ metadata: app.kubernetes.io/instance: ntfy app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: ntfy-attachments-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-ntfy-attachments-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-ntfy-attachments-remote-0.11.0 namespace: ntfy spec: suspend: false @@ -45,7 +45,7 @@ spec: - --s3-no-check-bucket - --rc - --rc-enable-metrics - - --metrics-addr=0.0.0.0:5572 + - --metrics-addr=0.0.0.0:5573 - --verbose env: - name: RCLONE_S3_PROVIDER @@ -106,6 +106,6 @@ spec: name: ntfy-attachments-rclone-destination-config - name: RCLONE_CONFIG_DEST_FORCE_PATH_STYLE value: "true" - image: rclone/rclone:1.74.0@sha256:d2e0e88359d0b2e67cfcd2c43d5405185eb8adfc207079df27c42da82c5207bc + image: rclone/rclone:1.74.1@sha256:eb8788b795f0009324e9572b0e2acb9b56885327c2746b07d67a9d3b893a6602 imagePullPolicy: IfNotPresent name: sync diff --git a/clusters/cl01tl/manifests/ntfy/ExternalSecret-ntfy-attachments-rclone-destination-config.yaml b/clusters/cl01tl/manifests/ntfy/ExternalSecret-ntfy-attachments-rclone-destination-config.yaml index c3a96ec70..4f5493ad6 100644 --- a/clusters/cl01tl/manifests/ntfy/ExternalSecret-ntfy-attachments-rclone-destination-config.yaml +++ b/clusters/cl01tl/manifests/ntfy/ExternalSecret-ntfy-attachments-rclone-destination-config.yaml @@ -4,10 +4,10 @@ metadata: name: ntfy-attachments-rclone-destination-config namespace: ntfy labels: - helm.sh/chart: rclone-ntfy-attachments-remote-0.9.1 + helm.sh/chart: rclone-ntfy-attachments-remote-0.11.0 app.kubernetes.io/instance: ntfy app.kubernetes.io/part-of: ntfy - app.kubernetes.io/version: "0.9.1" + app.kubernetes.io/version: "0.11.0" app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: ntfy-attachments-rclone-destination-config spec: diff --git a/clusters/cl01tl/manifests/ntfy/ExternalSecret-ntfy-attachments-rclone-source-config.yaml b/clusters/cl01tl/manifests/ntfy/ExternalSecret-ntfy-attachments-rclone-source-config.yaml index 46a5d3961..0ee1211b3 100644 --- a/clusters/cl01tl/manifests/ntfy/ExternalSecret-ntfy-attachments-rclone-source-config.yaml +++ b/clusters/cl01tl/manifests/ntfy/ExternalSecret-ntfy-attachments-rclone-source-config.yaml @@ -4,10 +4,10 @@ metadata: name: ntfy-attachments-rclone-source-config namespace: ntfy labels: - helm.sh/chart: rclone-ntfy-attachments-remote-0.9.1 + helm.sh/chart: rclone-ntfy-attachments-remote-0.11.0 app.kubernetes.io/instance: ntfy app.kubernetes.io/part-of: ntfy - app.kubernetes.io/version: "0.9.1" + app.kubernetes.io/version: "0.11.0" app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: ntfy-attachments-rclone-source-config spec: diff --git a/clusters/cl01tl/manifests/ntfy/PrometheusRule-ntfy-attachments-rclone.yaml b/clusters/cl01tl/manifests/ntfy/PrometheusRule-ntfy-attachments-rclone.yaml new file mode 100644 index 000000000..e7eef8c24 --- /dev/null +++ b/clusters/cl01tl/manifests/ntfy/PrometheusRule-ntfy-attachments-rclone.yaml @@ -0,0 +1,30 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: ntfy-attachments-rclone + namespace: ntfy + labels: + helm.sh/chart: rclone-ntfy-attachments-remote-0.11.0 + app.kubernetes.io/instance: ntfy + app.kubernetes.io/part-of: ntfy + app.kubernetes.io/version: "0.11.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: ntfy-attachments-rclone +spec: + groups: + - name: rclone.alerts + rules: + - alert: RclonePodFailed + expr: | + (kube_pod_container_status_last_terminated_exitcode > 0) + * on(pod, namespace) group_left(owner_name) + kube_pod_owner{owner_kind="Job", owner_name=~"rclone-.*"} + for: 1m + labels: + severity: critical + annotations: + summary: "Rclone Pod failed in {{ $labels.namespace }}" + description: | + A pod for the Rclone sync of s3 bucket 'ntfy-attachments' failed with exit code {{ $value }}. + Job: {{ $labels.owner_name }} + Namespace: {{ $labels.namespace }} diff --git a/clusters/cl01tl/manifests/ntfy/Service-ntfy-ntfy-attachments-rclone.yaml b/clusters/cl01tl/manifests/ntfy/Service-ntfy-ntfy-attachments-rclone.yaml index 2d06b45ab..e57e06f53 100644 --- a/clusters/cl01tl/manifests/ntfy/Service-ntfy-ntfy-attachments-rclone.yaml +++ b/clusters/cl01tl/manifests/ntfy/Service-ntfy-ntfy-attachments-rclone.yaml @@ -7,16 +7,24 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: ntfy-attachments-rclone app.kubernetes.io/service: ntfy-ntfy-attachments-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-ntfy-attachments-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-ntfy-attachments-remote-0.11.0 namespace: ntfy spec: type: ClusterIP ports: + - port: 5574 + targetPort: 5574 + protocol: TCP + name: metrics-prune + - port: 5573 + targetPort: 5573 + protocol: TCP + name: metrics-sync - port: 5572 targetPort: 5572 protocol: TCP - name: metrics + name: rc selector: app.kubernetes.io/controller: main app.kubernetes.io/instance: ntfy diff --git a/clusters/cl01tl/manifests/ntfy/ServiceAccount-ntfy-ntfy-attachments-rclone.yaml b/clusters/cl01tl/manifests/ntfy/ServiceAccount-ntfy-ntfy-attachments-rclone.yaml index fa719be86..d1fb8eb8a 100644 --- a/clusters/cl01tl/manifests/ntfy/ServiceAccount-ntfy-ntfy-attachments-rclone.yaml +++ b/clusters/cl01tl/manifests/ntfy/ServiceAccount-ntfy-ntfy-attachments-rclone.yaml @@ -6,6 +6,6 @@ metadata: app.kubernetes.io/instance: ntfy app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: ntfy-attachments-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-ntfy-attachments-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-ntfy-attachments-remote-0.11.0 namespace: ntfy diff --git a/clusters/cl01tl/manifests/ntfy/ServiceMonitor-ntfy-ntfy-attachments-rclone.yaml b/clusters/cl01tl/manifests/ntfy/ServiceMonitor-ntfy-ntfy-attachments-rclone.yaml index f226afb83..df85ad596 100644 --- a/clusters/cl01tl/manifests/ntfy/ServiceMonitor-ntfy-ntfy-attachments-rclone.yaml +++ b/clusters/cl01tl/manifests/ntfy/ServiceMonitor-ntfy-ntfy-attachments-rclone.yaml @@ -6,8 +6,8 @@ metadata: app.kubernetes.io/instance: ntfy app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: ntfy-attachments-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-ntfy-attachments-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-ntfy-attachments-remote-0.11.0 namespace: ntfy spec: jobLabel: app.kubernetes.io/name @@ -19,7 +19,11 @@ spec: app.kubernetes.io/instance: ntfy app.kubernetes.io/name: ntfy-attachments-rclone endpoints: - - interval: 30s + - interval: 10s path: /metrics - port: metrics - scrapeTimeout: 10s + port: metrics-sync + scrapeTimeout: 5s + - interval: 10s + path: /metrics + port: metrics-prune + scrapeTimeout: 5s diff --git a/clusters/cl01tl/manifests/openbao/CronJob-openbao-openbao-backups-external-rclone.yaml b/clusters/cl01tl/manifests/openbao/CronJob-openbao-openbao-backups-external-rclone.yaml index d75438082..24e869104 100644 --- a/clusters/cl01tl/manifests/openbao/CronJob-openbao-openbao-backups-external-rclone.yaml +++ b/clusters/cl01tl/manifests/openbao/CronJob-openbao-openbao-backups-external-rclone.yaml @@ -7,8 +7,8 @@ metadata: app.kubernetes.io/instance: openbao app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: openbao-backups-external-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-openbao-backups-external-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-openbao-backups-external-0.11.0 namespace: openbao spec: suspend: false @@ -46,7 +46,7 @@ spec: - --verbose - --rc - --rc-enable-metrics - - --metrics-addr=0.0.0.0:5572 + - --metrics-addr=0.0.0.0:5574 env: - name: RCLONE_CONFIG_DEST_TYPE value: s3 @@ -76,7 +76,7 @@ spec: name: openbao-backups-038053cd180284dc-rclone-destination-config - name: RCLONE_CONFIG_DEST_FORCE_PATH_STYLE value: "true" - image: rclone/rclone:1.74.0@sha256:d2e0e88359d0b2e67cfcd2c43d5405185eb8adfc207079df27c42da82c5207bc + image: rclone/rclone:1.74.1@sha256:eb8788b795f0009324e9572b0e2acb9b56885327c2746b07d67a9d3b893a6602 imagePullPolicy: IfNotPresent name: prune - args: @@ -88,7 +88,7 @@ spec: - 90d - --rc - --rc-enable-metrics - - --metrics-addr=0.0.0.0:5572 + - --metrics-addr=0.0.0.0:5573 - --verbose env: - name: RCLONE_S3_PROVIDER @@ -149,6 +149,6 @@ spec: name: openbao-backups-038053cd180284dc-rclone-destination-config - name: RCLONE_CONFIG_DEST_FORCE_PATH_STYLE value: "true" - image: rclone/rclone:1.74.0@sha256:d2e0e88359d0b2e67cfcd2c43d5405185eb8adfc207079df27c42da82c5207bc + image: rclone/rclone:1.74.1@sha256:eb8788b795f0009324e9572b0e2acb9b56885327c2746b07d67a9d3b893a6602 imagePullPolicy: IfNotPresent name: sync diff --git a/clusters/cl01tl/manifests/openbao/CronJob-openbao-openbao-backups-remote-rclone.yaml b/clusters/cl01tl/manifests/openbao/CronJob-openbao-openbao-backups-remote-rclone.yaml index 9942aceae..849589aa5 100644 --- a/clusters/cl01tl/manifests/openbao/CronJob-openbao-openbao-backups-remote-rclone.yaml +++ b/clusters/cl01tl/manifests/openbao/CronJob-openbao-openbao-backups-remote-rclone.yaml @@ -7,8 +7,8 @@ metadata: app.kubernetes.io/instance: openbao app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: openbao-backups-remote-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-openbao-backups-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-openbao-backups-remote-0.11.0 namespace: openbao spec: suspend: false @@ -46,7 +46,7 @@ spec: - --verbose - --rc - --rc-enable-metrics - - --metrics-addr=0.0.0.0:5572 + - --metrics-addr=0.0.0.0:5574 env: - name: RCLONE_CONFIG_DEST_TYPE value: s3 @@ -76,7 +76,7 @@ spec: name: openbao-backups-rclone-destination-config - name: RCLONE_CONFIG_DEST_FORCE_PATH_STYLE value: "true" - image: rclone/rclone:1.74.0@sha256:d2e0e88359d0b2e67cfcd2c43d5405185eb8adfc207079df27c42da82c5207bc + image: rclone/rclone:1.74.1@sha256:eb8788b795f0009324e9572b0e2acb9b56885327c2746b07d67a9d3b893a6602 imagePullPolicy: IfNotPresent name: prune - args: @@ -88,7 +88,7 @@ spec: - 90d - --rc - --rc-enable-metrics - - --metrics-addr=0.0.0.0:5572 + - --metrics-addr=0.0.0.0:5573 - --verbose env: - name: RCLONE_S3_PROVIDER @@ -149,6 +149,6 @@ spec: name: openbao-backups-rclone-destination-config - name: RCLONE_CONFIG_DEST_FORCE_PATH_STYLE value: "true" - image: rclone/rclone:1.74.0@sha256:d2e0e88359d0b2e67cfcd2c43d5405185eb8adfc207079df27c42da82c5207bc + image: rclone/rclone:1.74.1@sha256:eb8788b795f0009324e9572b0e2acb9b56885327c2746b07d67a9d3b893a6602 imagePullPolicy: IfNotPresent name: sync diff --git a/clusters/cl01tl/manifests/openbao/ExternalSecret-openbao-backups-038053cd180284dc-rclone-destination-config.yaml b/clusters/cl01tl/manifests/openbao/ExternalSecret-openbao-backups-038053cd180284dc-rclone-destination-config.yaml index 7ce6a72f0..bf664ed57 100644 --- a/clusters/cl01tl/manifests/openbao/ExternalSecret-openbao-backups-038053cd180284dc-rclone-destination-config.yaml +++ b/clusters/cl01tl/manifests/openbao/ExternalSecret-openbao-backups-038053cd180284dc-rclone-destination-config.yaml @@ -4,10 +4,10 @@ metadata: name: openbao-backups-038053cd180284dc-rclone-destination-config namespace: openbao labels: - helm.sh/chart: rclone-openbao-backups-external-0.9.1 + helm.sh/chart: rclone-openbao-backups-external-0.11.0 app.kubernetes.io/instance: openbao app.kubernetes.io/part-of: openbao - app.kubernetes.io/version: "0.9.1" + app.kubernetes.io/version: "0.11.0" app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: openbao-backups-038053cd180284dc-rclone-destination-config spec: diff --git a/clusters/cl01tl/manifests/openbao/ExternalSecret-openbao-backups-rclone-destination-config.yaml b/clusters/cl01tl/manifests/openbao/ExternalSecret-openbao-backups-rclone-destination-config.yaml index d54fa49ba..9a85eaaf2 100644 --- a/clusters/cl01tl/manifests/openbao/ExternalSecret-openbao-backups-rclone-destination-config.yaml +++ b/clusters/cl01tl/manifests/openbao/ExternalSecret-openbao-backups-rclone-destination-config.yaml @@ -4,10 +4,10 @@ metadata: name: openbao-backups-rclone-destination-config namespace: openbao labels: - helm.sh/chart: rclone-openbao-backups-remote-0.9.1 + helm.sh/chart: rclone-openbao-backups-remote-0.11.0 app.kubernetes.io/instance: openbao app.kubernetes.io/part-of: openbao - app.kubernetes.io/version: "0.9.1" + app.kubernetes.io/version: "0.11.0" app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: openbao-backups-rclone-destination-config spec: diff --git a/clusters/cl01tl/manifests/openbao/ExternalSecret-openbao-backups-rclone-source-config.yaml b/clusters/cl01tl/manifests/openbao/ExternalSecret-openbao-backups-rclone-source-config.yaml index 38a0a2f77..213e090db 100644 --- a/clusters/cl01tl/manifests/openbao/ExternalSecret-openbao-backups-rclone-source-config.yaml +++ b/clusters/cl01tl/manifests/openbao/ExternalSecret-openbao-backups-rclone-source-config.yaml @@ -4,10 +4,10 @@ metadata: name: openbao-backups-rclone-source-config namespace: openbao labels: - helm.sh/chart: rclone-openbao-backups-remote-0.9.1 + helm.sh/chart: rclone-openbao-backups-remote-0.11.0 app.kubernetes.io/instance: openbao app.kubernetes.io/part-of: openbao - app.kubernetes.io/version: "0.9.1" + app.kubernetes.io/version: "0.11.0" app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: openbao-backups-rclone-source-config spec: diff --git a/clusters/cl01tl/manifests/openbao/PrometheusRule-openbao-backups-external-rclone.yaml b/clusters/cl01tl/manifests/openbao/PrometheusRule-openbao-backups-external-rclone.yaml new file mode 100644 index 000000000..e7b873c4b --- /dev/null +++ b/clusters/cl01tl/manifests/openbao/PrometheusRule-openbao-backups-external-rclone.yaml @@ -0,0 +1,30 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: openbao-backups-external-rclone + namespace: openbao + labels: + helm.sh/chart: rclone-openbao-backups-external-0.11.0 + app.kubernetes.io/instance: openbao + app.kubernetes.io/part-of: openbao + app.kubernetes.io/version: "0.11.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: openbao-backups-external-rclone +spec: + groups: + - name: rclone.alerts + rules: + - alert: RclonePodFailed + expr: | + (kube_pod_container_status_last_terminated_exitcode > 0) + * on(pod, namespace) group_left(owner_name) + kube_pod_owner{owner_kind="Job", owner_name=~"rclone-.*"} + for: 1m + labels: + severity: critical + annotations: + summary: "Rclone Pod failed in {{ $labels.namespace }}" + description: | + A pod for the Rclone sync of s3 bucket 'openbao-backups' failed with exit code {{ $value }}. + Job: {{ $labels.owner_name }} + Namespace: {{ $labels.namespace }} diff --git a/clusters/cl01tl/manifests/openbao/PrometheusRule-openbao-backups-remote-rclone.yaml b/clusters/cl01tl/manifests/openbao/PrometheusRule-openbao-backups-remote-rclone.yaml new file mode 100644 index 000000000..3a38c9b3d --- /dev/null +++ b/clusters/cl01tl/manifests/openbao/PrometheusRule-openbao-backups-remote-rclone.yaml @@ -0,0 +1,30 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: openbao-backups-remote-rclone + namespace: openbao + labels: + helm.sh/chart: rclone-openbao-backups-remote-0.11.0 + app.kubernetes.io/instance: openbao + app.kubernetes.io/part-of: openbao + app.kubernetes.io/version: "0.11.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: openbao-backups-remote-rclone +spec: + groups: + - name: rclone.alerts + rules: + - alert: RclonePodFailed + expr: | + (kube_pod_container_status_last_terminated_exitcode > 0) + * on(pod, namespace) group_left(owner_name) + kube_pod_owner{owner_kind="Job", owner_name=~"rclone-.*"} + for: 1m + labels: + severity: critical + annotations: + summary: "Rclone Pod failed in {{ $labels.namespace }}" + description: | + A pod for the Rclone sync of s3 bucket 'openbao-backups' failed with exit code {{ $value }}. + Job: {{ $labels.owner_name }} + Namespace: {{ $labels.namespace }} diff --git a/clusters/cl01tl/manifests/openbao/Service-openbao-openbao-backups-external-rclone.yaml b/clusters/cl01tl/manifests/openbao/Service-openbao-openbao-backups-external-rclone.yaml index 20f805f1b..eb0736772 100644 --- a/clusters/cl01tl/manifests/openbao/Service-openbao-openbao-backups-external-rclone.yaml +++ b/clusters/cl01tl/manifests/openbao/Service-openbao-openbao-backups-external-rclone.yaml @@ -7,16 +7,24 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: openbao-backups-external-rclone app.kubernetes.io/service: openbao-openbao-backups-external-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-openbao-backups-external-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-openbao-backups-external-0.11.0 namespace: openbao spec: type: ClusterIP ports: + - port: 5574 + targetPort: 5574 + protocol: TCP + name: metrics-prune + - port: 5573 + targetPort: 5573 + protocol: TCP + name: metrics-sync - port: 5572 targetPort: 5572 protocol: TCP - name: metrics + name: rc selector: app.kubernetes.io/controller: main app.kubernetes.io/instance: openbao diff --git a/clusters/cl01tl/manifests/openbao/Service-openbao-openbao-backups-remote-rclone.yaml b/clusters/cl01tl/manifests/openbao/Service-openbao-openbao-backups-remote-rclone.yaml index 713cdb623..ad7ba2dcf 100644 --- a/clusters/cl01tl/manifests/openbao/Service-openbao-openbao-backups-remote-rclone.yaml +++ b/clusters/cl01tl/manifests/openbao/Service-openbao-openbao-backups-remote-rclone.yaml @@ -7,16 +7,24 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: openbao-backups-remote-rclone app.kubernetes.io/service: openbao-openbao-backups-remote-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-openbao-backups-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-openbao-backups-remote-0.11.0 namespace: openbao spec: type: ClusterIP ports: + - port: 5574 + targetPort: 5574 + protocol: TCP + name: metrics-prune + - port: 5573 + targetPort: 5573 + protocol: TCP + name: metrics-sync - port: 5572 targetPort: 5572 protocol: TCP - name: metrics + name: rc selector: app.kubernetes.io/controller: main app.kubernetes.io/instance: openbao diff --git a/clusters/cl01tl/manifests/openbao/ServiceAccount-openbao-openbao-backups-external-rclone.yaml b/clusters/cl01tl/manifests/openbao/ServiceAccount-openbao-openbao-backups-external-rclone.yaml index a880d5d18..2bc29eae0 100644 --- a/clusters/cl01tl/manifests/openbao/ServiceAccount-openbao-openbao-backups-external-rclone.yaml +++ b/clusters/cl01tl/manifests/openbao/ServiceAccount-openbao-openbao-backups-external-rclone.yaml @@ -6,6 +6,6 @@ metadata: app.kubernetes.io/instance: openbao app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: openbao-backups-external-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-openbao-backups-external-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-openbao-backups-external-0.11.0 namespace: openbao diff --git a/clusters/cl01tl/manifests/openbao/ServiceAccount-openbao-openbao-backups-remote-rclone.yaml b/clusters/cl01tl/manifests/openbao/ServiceAccount-openbao-openbao-backups-remote-rclone.yaml index 34fa0e8ac..ab5704997 100644 --- a/clusters/cl01tl/manifests/openbao/ServiceAccount-openbao-openbao-backups-remote-rclone.yaml +++ b/clusters/cl01tl/manifests/openbao/ServiceAccount-openbao-openbao-backups-remote-rclone.yaml @@ -6,6 +6,6 @@ metadata: app.kubernetes.io/instance: openbao app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: openbao-backups-remote-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-openbao-backups-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-openbao-backups-remote-0.11.0 namespace: openbao diff --git a/clusters/cl01tl/manifests/openbao/ServiceMonitor-openbao-openbao-backups-external-rclone.yaml b/clusters/cl01tl/manifests/openbao/ServiceMonitor-openbao-openbao-backups-external-rclone.yaml index c3190d6c9..153d0d7c2 100644 --- a/clusters/cl01tl/manifests/openbao/ServiceMonitor-openbao-openbao-backups-external-rclone.yaml +++ b/clusters/cl01tl/manifests/openbao/ServiceMonitor-openbao-openbao-backups-external-rclone.yaml @@ -6,8 +6,8 @@ metadata: app.kubernetes.io/instance: openbao app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: openbao-backups-external-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-openbao-backups-external-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-openbao-backups-external-0.11.0 namespace: openbao spec: jobLabel: app.kubernetes.io/name @@ -19,7 +19,11 @@ spec: app.kubernetes.io/instance: openbao app.kubernetes.io/name: openbao-backups-external-rclone endpoints: - - interval: 30s + - interval: 10s path: /metrics - port: metrics - scrapeTimeout: 10s + port: metrics-sync + scrapeTimeout: 5s + - interval: 10s + path: /metrics + port: metrics-prune + scrapeTimeout: 5s diff --git a/clusters/cl01tl/manifests/openbao/ServiceMonitor-openbao-openbao-backups-remote-rclone.yaml b/clusters/cl01tl/manifests/openbao/ServiceMonitor-openbao-openbao-backups-remote-rclone.yaml index 1bf00ab92..4d16bfb5c 100644 --- a/clusters/cl01tl/manifests/openbao/ServiceMonitor-openbao-openbao-backups-remote-rclone.yaml +++ b/clusters/cl01tl/manifests/openbao/ServiceMonitor-openbao-openbao-backups-remote-rclone.yaml @@ -6,8 +6,8 @@ metadata: app.kubernetes.io/instance: openbao app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: openbao-backups-remote-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-openbao-backups-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-openbao-backups-remote-0.11.0 namespace: openbao spec: jobLabel: app.kubernetes.io/name @@ -19,7 +19,11 @@ spec: app.kubernetes.io/instance: openbao app.kubernetes.io/name: openbao-backups-remote-rclone endpoints: - - interval: 30s + - interval: 10s path: /metrics - port: metrics - scrapeTimeout: 10s + port: metrics-sync + scrapeTimeout: 5s + - interval: 10s + path: /metrics + port: metrics-prune + scrapeTimeout: 5s diff --git a/clusters/cl01tl/manifests/rclone/CronJob-rclone-web-assets-remote-rclone.yaml b/clusters/cl01tl/manifests/rclone/CronJob-rclone-web-assets-remote-rclone.yaml index 68a991b0f..dfa5b1c50 100644 --- a/clusters/cl01tl/manifests/rclone/CronJob-rclone-web-assets-remote-rclone.yaml +++ b/clusters/cl01tl/manifests/rclone/CronJob-rclone-web-assets-remote-rclone.yaml @@ -7,8 +7,8 @@ metadata: app.kubernetes.io/instance: rclone app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: web-assets-remote-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-web-assets-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-web-assets-remote-0.11.0 namespace: rclone spec: suspend: false @@ -45,7 +45,7 @@ spec: - --s3-no-check-bucket - --rc - --rc-enable-metrics - - --metrics-addr=0.0.0.0:5572 + - --metrics-addr=0.0.0.0:5573 - --verbose env: - name: RCLONE_S3_PROVIDER @@ -106,6 +106,6 @@ spec: name: web-assets-rclone-destination-config - name: RCLONE_CONFIG_DEST_FORCE_PATH_STYLE value: "true" - image: rclone/rclone:1.74.0@sha256:d2e0e88359d0b2e67cfcd2c43d5405185eb8adfc207079df27c42da82c5207bc + image: rclone/rclone:1.74.1@sha256:eb8788b795f0009324e9572b0e2acb9b56885327c2746b07d67a9d3b893a6602 imagePullPolicy: IfNotPresent name: sync diff --git a/clusters/cl01tl/manifests/rclone/ExternalSecret-web-assets-770aef58c931fcf4-rclone-source-config.yaml b/clusters/cl01tl/manifests/rclone/ExternalSecret-web-assets-770aef58c931fcf4-rclone-source-config.yaml index e3242654b..757e0a5e6 100644 --- a/clusters/cl01tl/manifests/rclone/ExternalSecret-web-assets-770aef58c931fcf4-rclone-source-config.yaml +++ b/clusters/cl01tl/manifests/rclone/ExternalSecret-web-assets-770aef58c931fcf4-rclone-source-config.yaml @@ -4,10 +4,10 @@ metadata: name: web-assets-770aef58c931fcf4-rclone-source-config namespace: rclone labels: - helm.sh/chart: rclone-web-assets-remote-0.9.1 + helm.sh/chart: rclone-web-assets-remote-0.11.0 app.kubernetes.io/instance: rclone app.kubernetes.io/part-of: rclone - app.kubernetes.io/version: "0.9.1" + app.kubernetes.io/version: "0.11.0" app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: web-assets-770aef58c931fcf4-rclone-source-config spec: diff --git a/clusters/cl01tl/manifests/rclone/ExternalSecret-web-assets-rclone-destination-config.yaml b/clusters/cl01tl/manifests/rclone/ExternalSecret-web-assets-rclone-destination-config.yaml index 52681aa6e..e231dd8e3 100644 --- a/clusters/cl01tl/manifests/rclone/ExternalSecret-web-assets-rclone-destination-config.yaml +++ b/clusters/cl01tl/manifests/rclone/ExternalSecret-web-assets-rclone-destination-config.yaml @@ -4,10 +4,10 @@ metadata: name: web-assets-rclone-destination-config namespace: rclone labels: - helm.sh/chart: rclone-web-assets-remote-0.9.1 + helm.sh/chart: rclone-web-assets-remote-0.11.0 app.kubernetes.io/instance: rclone app.kubernetes.io/part-of: rclone - app.kubernetes.io/version: "0.9.1" + app.kubernetes.io/version: "0.11.0" app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: web-assets-rclone-destination-config spec: diff --git a/clusters/cl01tl/manifests/rclone/PrometheusRule-web-assets-remote-rclone.yaml b/clusters/cl01tl/manifests/rclone/PrometheusRule-web-assets-remote-rclone.yaml new file mode 100644 index 000000000..211196e29 --- /dev/null +++ b/clusters/cl01tl/manifests/rclone/PrometheusRule-web-assets-remote-rclone.yaml @@ -0,0 +1,30 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: web-assets-remote-rclone + namespace: rclone + labels: + helm.sh/chart: rclone-web-assets-remote-0.11.0 + app.kubernetes.io/instance: rclone + app.kubernetes.io/part-of: rclone + app.kubernetes.io/version: "0.11.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: web-assets-remote-rclone +spec: + groups: + - name: rclone.alerts + rules: + - alert: RclonePodFailed + expr: | + (kube_pod_container_status_last_terminated_exitcode > 0) + * on(pod, namespace) group_left(owner_name) + kube_pod_owner{owner_kind="Job", owner_name=~"rclone-.*"} + for: 1m + labels: + severity: critical + annotations: + summary: "Rclone Pod failed in {{ $labels.namespace }}" + description: | + A pod for the Rclone sync of s3 bucket 'web-assets-770aef58c931fcf4' failed with exit code {{ $value }}. + Job: {{ $labels.owner_name }} + Namespace: {{ $labels.namespace }} diff --git a/clusters/cl01tl/manifests/rclone/Service-rclone-web-assets-remote-rclone.yaml b/clusters/cl01tl/manifests/rclone/Service-rclone-web-assets-remote-rclone.yaml index f74a7b952..373943949 100644 --- a/clusters/cl01tl/manifests/rclone/Service-rclone-web-assets-remote-rclone.yaml +++ b/clusters/cl01tl/manifests/rclone/Service-rclone-web-assets-remote-rclone.yaml @@ -7,16 +7,24 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: web-assets-remote-rclone app.kubernetes.io/service: rclone-web-assets-remote-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-web-assets-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-web-assets-remote-0.11.0 namespace: rclone spec: type: ClusterIP ports: + - port: 5574 + targetPort: 5574 + protocol: TCP + name: metrics-prune + - port: 5573 + targetPort: 5573 + protocol: TCP + name: metrics-sync - port: 5572 targetPort: 5572 protocol: TCP - name: metrics + name: rc selector: app.kubernetes.io/controller: main app.kubernetes.io/instance: rclone diff --git a/clusters/cl01tl/manifests/rclone/ServiceAccount-rclone-web-assets-remote-rclone.yaml b/clusters/cl01tl/manifests/rclone/ServiceAccount-rclone-web-assets-remote-rclone.yaml index 75efa5e8e..3ac925c93 100644 --- a/clusters/cl01tl/manifests/rclone/ServiceAccount-rclone-web-assets-remote-rclone.yaml +++ b/clusters/cl01tl/manifests/rclone/ServiceAccount-rclone-web-assets-remote-rclone.yaml @@ -6,6 +6,6 @@ metadata: app.kubernetes.io/instance: rclone app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: web-assets-remote-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-web-assets-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-web-assets-remote-0.11.0 namespace: rclone diff --git a/clusters/cl01tl/manifests/rclone/ServiceMonitor-rclone-web-assets-remote-rclone.yaml b/clusters/cl01tl/manifests/rclone/ServiceMonitor-rclone-web-assets-remote-rclone.yaml index 4a4f5729b..202387db2 100644 --- a/clusters/cl01tl/manifests/rclone/ServiceMonitor-rclone-web-assets-remote-rclone.yaml +++ b/clusters/cl01tl/manifests/rclone/ServiceMonitor-rclone-web-assets-remote-rclone.yaml @@ -6,8 +6,8 @@ metadata: app.kubernetes.io/instance: rclone app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: web-assets-remote-rclone - app.kubernetes.io/version: v1.74.0 - helm.sh/chart: rclone-web-assets-remote-0.9.1 + app.kubernetes.io/version: v1.74.1 + helm.sh/chart: rclone-web-assets-remote-0.11.0 namespace: rclone spec: jobLabel: app.kubernetes.io/name @@ -19,7 +19,11 @@ spec: app.kubernetes.io/instance: rclone app.kubernetes.io/name: web-assets-remote-rclone endpoints: - - interval: 30s + - interval: 10s path: /metrics - port: metrics - scrapeTimeout: 10s + port: metrics-sync + scrapeTimeout: 5s + - interval: 10s + path: /metrics + port: metrics-prune + scrapeTimeout: 5s