1026 lines
35 KiB
YAML
1026 lines
35 KiB
YAML
---
|
|
# Source: immich/templates/persistent-volume.yaml
|
|
apiVersion: v1
|
|
kind: PersistentVolume
|
|
metadata:
|
|
name: immich-nfs-storage
|
|
namespace: immich
|
|
labels:
|
|
app.kubernetes.io/name: immich-nfs-storage
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/part-of: immich
|
|
spec:
|
|
persistentVolumeReclaimPolicy: Retain
|
|
storageClassName: nfs-client
|
|
capacity:
|
|
storage: 1Gi
|
|
accessModes:
|
|
- ReadWriteMany
|
|
nfs:
|
|
path: /volume2/Storage/Immich
|
|
server: synologybond.alexlebens.net
|
|
mountOptions:
|
|
- vers=4
|
|
- minorversion=1
|
|
- noac
|
|
---
|
|
# Source: immich/charts/immich/templates/common.yaml
|
|
---
|
|
kind: PersistentVolumeClaim
|
|
apiVersion: v1
|
|
metadata:
|
|
name: immich
|
|
labels:
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/name: immich
|
|
helm.sh/chart: immich-4.4.0
|
|
annotations:
|
|
helm.sh/resource-policy: keep
|
|
namespace: immich
|
|
spec:
|
|
accessModes:
|
|
- "ReadWriteOnce"
|
|
resources:
|
|
requests:
|
|
storage: "10Gi"
|
|
storageClassName: "ceph-block"
|
|
---
|
|
# Source: immich/templates/persistent-volume-claim.yaml
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: immich-nfs-storage
|
|
namespace: immich
|
|
labels:
|
|
app.kubernetes.io/name: immich-nfs-storage
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/part-of: immich
|
|
spec:
|
|
volumeName: immich-nfs-storage
|
|
storageClassName: nfs-client
|
|
accessModes:
|
|
- ReadWriteMany
|
|
resources:
|
|
requests:
|
|
storage: 1Gi
|
|
---
|
|
# Source: immich/charts/immich/templates/common.yaml
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: immich-machine-learning
|
|
labels:
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/name: immich
|
|
app.kubernetes.io/service: immich-machine-learning
|
|
helm.sh/chart: immich-4.4.0
|
|
namespace: immich
|
|
spec:
|
|
type: ClusterIP
|
|
ports:
|
|
- port: 3003
|
|
targetPort: 3003
|
|
protocol: TCP
|
|
name: http
|
|
selector:
|
|
app.kubernetes.io/controller: machine-learning
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/name: immich
|
|
---
|
|
# Source: immich/charts/immich/templates/common.yaml
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: immich-main
|
|
labels:
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/name: immich
|
|
app.kubernetes.io/service: immich-main
|
|
helm.sh/chart: immich-4.4.0
|
|
namespace: immich
|
|
spec:
|
|
type: ClusterIP
|
|
ports:
|
|
- port: 2283
|
|
targetPort: 2283
|
|
protocol: TCP
|
|
name: http
|
|
- port: 8081
|
|
targetPort: 8081
|
|
protocol: TCP
|
|
name: metrics-api
|
|
- port: 8082
|
|
targetPort: 8082
|
|
protocol: TCP
|
|
name: metrics-ms
|
|
selector:
|
|
app.kubernetes.io/controller: main
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/name: immich
|
|
---
|
|
# Source: immich/charts/immich/templates/common.yaml
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: immich-machine-learning
|
|
labels:
|
|
app.kubernetes.io/controller: machine-learning
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/name: immich
|
|
helm.sh/chart: immich-4.4.0
|
|
namespace: immich
|
|
spec:
|
|
revisionHistoryLimit: 3
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/controller: machine-learning
|
|
app.kubernetes.io/name: immich
|
|
app.kubernetes.io/instance: immich
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/controller: machine-learning
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/name: immich
|
|
spec:
|
|
enableServiceLinks: false
|
|
serviceAccountName: default
|
|
automountServiceAccountToken: true
|
|
hostIPC: false
|
|
hostNetwork: false
|
|
hostPID: false
|
|
dnsPolicy: ClusterFirst
|
|
containers:
|
|
- env:
|
|
- name: TRANSFORMERS_CACHE
|
|
value: /cache
|
|
image: ghcr.io/immich-app/immich-machine-learning:v2.3.1
|
|
imagePullPolicy: IfNotPresent
|
|
livenessProbe:
|
|
failureThreshold: 3
|
|
httpGet:
|
|
path: /ping
|
|
port: 3003
|
|
initialDelaySeconds: 0
|
|
periodSeconds: 10
|
|
timeoutSeconds: 1
|
|
name: main
|
|
readinessProbe:
|
|
failureThreshold: 3
|
|
httpGet:
|
|
path: /ping
|
|
port: 3003
|
|
initialDelaySeconds: 0
|
|
periodSeconds: 10
|
|
timeoutSeconds: 1
|
|
resources:
|
|
limits:
|
|
gpu.intel.com/i915: 1
|
|
requests:
|
|
cpu: 10m
|
|
gpu.intel.com/i915: 1
|
|
memory: 256Mi
|
|
volumeMounts:
|
|
- mountPath: /cache
|
|
name: cache
|
|
volumes:
|
|
- name: cache
|
|
persistentVolumeClaim:
|
|
claimName: immich
|
|
---
|
|
# Source: immich/charts/immich/templates/common.yaml
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: immich-main
|
|
labels:
|
|
app.kubernetes.io/controller: main
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/name: immich
|
|
helm.sh/chart: immich-4.4.0
|
|
namespace: immich
|
|
spec:
|
|
revisionHistoryLimit: 3
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/controller: main
|
|
app.kubernetes.io/name: immich
|
|
app.kubernetes.io/instance: immich
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/controller: main
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/name: immich
|
|
spec:
|
|
enableServiceLinks: false
|
|
serviceAccountName: default
|
|
automountServiceAccountToken: true
|
|
hostIPC: false
|
|
hostNetwork: false
|
|
hostPID: false
|
|
dnsPolicy: ClusterFirst
|
|
containers:
|
|
- env:
|
|
- name: TZ
|
|
value: US/Central
|
|
- name: IMMICH_TELEMETRY_INCLUDE
|
|
value: all
|
|
- name: IMMICH_CONFIG_FILE
|
|
value: /config/immich.json
|
|
- name: IMMICH_MACHINE_LEARNING_URL
|
|
value: http://immich-machine-learning.immich:3003
|
|
- name: REDIS_HOSTNAME
|
|
value: redis-replication-immich-master
|
|
- name: DB_VECTOR_EXTENSION
|
|
value: vectorchord
|
|
- name: DB_HOSTNAME
|
|
valueFrom:
|
|
secretKeyRef:
|
|
key: host
|
|
name: immich-postgresql-17-cluster-app
|
|
- name: DB_DATABASE_NAME
|
|
valueFrom:
|
|
secretKeyRef:
|
|
key: dbname
|
|
name: immich-postgresql-17-cluster-app
|
|
- name: DB_PORT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
key: port
|
|
name: immich-postgresql-17-cluster-app
|
|
- name: DB_USERNAME
|
|
valueFrom:
|
|
secretKeyRef:
|
|
key: user
|
|
name: immich-postgresql-17-cluster-app
|
|
- name: DB_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
key: password
|
|
name: immich-postgresql-17-cluster-app
|
|
image: ghcr.io/immich-app/immich-server:v2.3.1
|
|
imagePullPolicy: IfNotPresent
|
|
livenessProbe:
|
|
failureThreshold: 3
|
|
httpGet:
|
|
path: /api/server/ping
|
|
port: 2283
|
|
initialDelaySeconds: 0
|
|
periodSeconds: 10
|
|
timeoutSeconds: 1
|
|
name: main
|
|
readinessProbe:
|
|
failureThreshold: 3
|
|
httpGet:
|
|
path: /api/server/ping
|
|
port: 2283
|
|
initialDelaySeconds: 0
|
|
periodSeconds: 10
|
|
timeoutSeconds: 1
|
|
resources:
|
|
limits:
|
|
gpu.intel.com/i915: 1
|
|
requests:
|
|
cpu: 10m
|
|
gpu.intel.com/i915: 1
|
|
memory: 512Mi
|
|
startupProbe:
|
|
failureThreshold: 30
|
|
httpGet:
|
|
path: /api/server/ping
|
|
port: 2283
|
|
initialDelaySeconds: 0
|
|
periodSeconds: 10
|
|
timeoutSeconds: 1
|
|
volumeMounts:
|
|
- mountPath: /config/immich.json
|
|
mountPropagation: None
|
|
name: config
|
|
readOnly: true
|
|
subPath: immich.json
|
|
- mountPath: /usr/src/app/upload
|
|
name: media
|
|
volumes:
|
|
- name: config
|
|
secret:
|
|
secretName: immich-config-secret
|
|
- name: media
|
|
persistentVolumeClaim:
|
|
claimName: immich-nfs-storage
|
|
---
|
|
# Source: immich/charts/postgres-17-cluster/templates/cluster.yaml
|
|
apiVersion: postgresql.cnpg.io/v1
|
|
kind: Cluster
|
|
metadata:
|
|
name: immich-postgresql-17-cluster
|
|
namespace: immich
|
|
labels:
|
|
helm.sh/chart: postgres-17-cluster-6.16.0
|
|
app.kubernetes.io/name: immich-postgresql-17
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/part-of: immich
|
|
app.kubernetes.io/version: "6.16.0"
|
|
app.kubernetes.io/managed-by: Helm
|
|
spec:
|
|
instances: 3
|
|
imageName: "ghcr.io/tensorchord/cloudnative-vectorchord:17.5-0.4.3"
|
|
imagePullPolicy: IfNotPresent
|
|
postgresUID: 26
|
|
postgresGID: 26
|
|
plugins:
|
|
- name: barman-cloud.cloudnative-pg.io
|
|
enabled: true
|
|
isWALArchiver: false
|
|
parameters:
|
|
barmanObjectName: "immich-postgresql-17-external-backup"
|
|
serverName: "immich-postgresql-17-backup-2"
|
|
- name: barman-cloud.cloudnative-pg.io
|
|
enabled: true
|
|
isWALArchiver: true
|
|
parameters:
|
|
barmanObjectName: "immich-postgresql-17-garage-local-backup"
|
|
serverName: "immich-postgresql-17-backup-1"
|
|
|
|
externalClusters:
|
|
- name: recovery
|
|
plugin:
|
|
name: barman-cloud.cloudnative-pg.io
|
|
parameters:
|
|
barmanObjectName: "immich-postgresql-17-recovery"
|
|
serverName: immich-postgresql-17-backup-1
|
|
|
|
storage:
|
|
size: 10Gi
|
|
storageClass: local-path
|
|
walStorage:
|
|
size: 2Gi
|
|
storageClass: local-path
|
|
resources:
|
|
limits:
|
|
hugepages-2Mi: 256Mi
|
|
requests:
|
|
cpu: 100m
|
|
memory: 256Mi
|
|
|
|
affinity:
|
|
enablePodAntiAffinity: true
|
|
topologyKey: kubernetes.io/hostname
|
|
|
|
primaryUpdateMethod: switchover
|
|
primaryUpdateStrategy: unsupervised
|
|
logLevel: info
|
|
enableSuperuserAccess: false
|
|
enablePDB: true
|
|
|
|
postgresql:
|
|
shared_preload_libraries:
|
|
- vchord.so
|
|
parameters:
|
|
hot_standby_feedback: "on"
|
|
max_slot_wal_keep_size: 2000MB
|
|
shared_buffers: 256MB
|
|
|
|
monitoring:
|
|
enablePodMonitor: true
|
|
disableDefaultQueries: false
|
|
|
|
|
|
bootstrap:
|
|
recovery:
|
|
|
|
database: app
|
|
|
|
source: immich-postgresql-17-backup-1
|
|
|
|
externalClusters:
|
|
- name: immich-postgresql-17-backup-1
|
|
plugin:
|
|
name: barman-cloud.cloudnative-pg.io
|
|
enabled: true
|
|
isWALArchiver: false
|
|
parameters:
|
|
barmanObjectName: "immich-postgresql-17-recovery"
|
|
serverName: immich-postgresql-17-backup-1
|
|
---
|
|
# Source: immich/templates/external-secrets.yaml
|
|
apiVersion: external-secrets.io/v1
|
|
kind: ExternalSecret
|
|
metadata:
|
|
name: immich-config-secret
|
|
namespace: immich
|
|
labels:
|
|
app.kubernetes.io/name: immich-config-secret
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/part-of: immich
|
|
spec:
|
|
secretStoreRef:
|
|
kind: ClusterSecretStore
|
|
name: vault
|
|
data:
|
|
- secretKey: immich.json
|
|
remoteRef:
|
|
conversionStrategy: Default
|
|
decodingStrategy: None
|
|
key: /cl01tl/immich/config
|
|
metadataPolicy: None
|
|
property: immich.json
|
|
---
|
|
# Source: immich/templates/external-secrets.yaml
|
|
apiVersion: external-secrets.io/v1
|
|
kind: ExternalSecret
|
|
metadata:
|
|
name: immich-postgresql-17-cluster-backup-secret
|
|
namespace: immich
|
|
labels:
|
|
app.kubernetes.io/name: immich-postgresql-17-cluster-backup-secret
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/part-of: immich
|
|
spec:
|
|
secretStoreRef:
|
|
kind: ClusterSecretStore
|
|
name: vault
|
|
data:
|
|
- secretKey: ACCESS_KEY_ID
|
|
remoteRef:
|
|
conversionStrategy: Default
|
|
decodingStrategy: None
|
|
key: /digital-ocean/home-infra/postgres-backups
|
|
metadataPolicy: None
|
|
property: access
|
|
- secretKey: ACCESS_SECRET_KEY
|
|
remoteRef:
|
|
conversionStrategy: Default
|
|
decodingStrategy: None
|
|
key: /digital-ocean/home-infra/postgres-backups
|
|
metadataPolicy: None
|
|
property: secret
|
|
---
|
|
# Source: immich/templates/external-secrets.yaml
|
|
apiVersion: external-secrets.io/v1
|
|
kind: ExternalSecret
|
|
metadata:
|
|
name: immich-postgresql-17-cluster-backup-secret-garage
|
|
namespace: immich
|
|
labels:
|
|
app.kubernetes.io/name: immich-postgresql-17-cluster-backup-secret-garage
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/part-of: immich
|
|
spec:
|
|
secretStoreRef:
|
|
kind: ClusterSecretStore
|
|
name: vault
|
|
data:
|
|
- secretKey: ACCESS_KEY_ID
|
|
remoteRef:
|
|
conversionStrategy: Default
|
|
decodingStrategy: None
|
|
key: /garage/home-infra/postgres-backups
|
|
metadataPolicy: None
|
|
property: ACCESS_KEY_ID
|
|
- secretKey: ACCESS_SECRET_KEY
|
|
remoteRef:
|
|
conversionStrategy: Default
|
|
decodingStrategy: None
|
|
key: /garage/home-infra/postgres-backups
|
|
metadataPolicy: None
|
|
property: ACCESS_SECRET_KEY
|
|
- secretKey: ACCESS_REGION
|
|
remoteRef:
|
|
conversionStrategy: Default
|
|
decodingStrategy: None
|
|
key: /garage/home-infra/postgres-backups
|
|
metadataPolicy: None
|
|
property: ACCESS_REGION
|
|
---
|
|
# Source: immich/templates/http-route.yaml
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: http-route-immich
|
|
namespace: immich
|
|
labels:
|
|
app.kubernetes.io/name: http-route-immich
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/part-of: immich
|
|
spec:
|
|
parentRefs:
|
|
- group: gateway.networking.k8s.io
|
|
kind: Gateway
|
|
name: traefik-gateway
|
|
namespace: traefik
|
|
hostnames:
|
|
- immich.alexlebens.net
|
|
rules:
|
|
- matches:
|
|
- path:
|
|
type: PathPrefix
|
|
value: /
|
|
backendRefs:
|
|
- group: ''
|
|
kind: Service
|
|
name: immich-main
|
|
port: 2283
|
|
weight: 100
|
|
---
|
|
# Source: immich/charts/postgres-17-cluster/templates/object-store.yaml
|
|
apiVersion: barmancloud.cnpg.io/v1
|
|
kind: ObjectStore
|
|
metadata:
|
|
name: "immich-postgresql-17-external-backup"
|
|
namespace: immich
|
|
labels:
|
|
helm.sh/chart: postgres-17-cluster-6.16.0
|
|
app.kubernetes.io/name: immich-postgresql-17
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/part-of: immich
|
|
app.kubernetes.io/version: "6.16.0"
|
|
app.kubernetes.io/managed-by: Helm
|
|
spec:
|
|
retentionPolicy: 30d
|
|
configuration:
|
|
destinationPath: s3://postgres-backups-ce540ddf106d186bbddca68a/cl01tl/immich/immich-postgresql-17-cluster
|
|
endpointURL: https://nyc3.digitaloceanspaces.com
|
|
s3Credentials:
|
|
accessKeyId:
|
|
name: immich-postgresql-17-cluster-backup-secret
|
|
key: ACCESS_KEY_ID
|
|
secretAccessKey:
|
|
name: immich-postgresql-17-cluster-backup-secret
|
|
key: ACCESS_SECRET_KEY
|
|
---
|
|
# Source: immich/charts/postgres-17-cluster/templates/object-store.yaml
|
|
apiVersion: barmancloud.cnpg.io/v1
|
|
kind: ObjectStore
|
|
metadata:
|
|
name: "immich-postgresql-17-garage-local-backup"
|
|
namespace: immich
|
|
labels:
|
|
helm.sh/chart: postgres-17-cluster-6.16.0
|
|
app.kubernetes.io/name: immich-postgresql-17
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/part-of: immich
|
|
app.kubernetes.io/version: "6.16.0"
|
|
app.kubernetes.io/managed-by: Helm
|
|
spec:
|
|
retentionPolicy: 3d
|
|
configuration:
|
|
destinationPath: s3://postgres-backups/cl01tl/immich/immich-postgresql-17-cluster
|
|
endpointURL: http://garage-main.garage:3900
|
|
s3Credentials:
|
|
accessKeyId:
|
|
name: immich-postgresql-17-cluster-backup-secret-garage
|
|
key: ACCESS_KEY_ID
|
|
secretAccessKey:
|
|
name: immich-postgresql-17-cluster-backup-secret-garage
|
|
key: ACCESS_SECRET_KEY
|
|
region:
|
|
name: immich-postgresql-17-cluster-backup-secret-garage
|
|
key: ACCESS_REGION
|
|
---
|
|
# Source: immich/charts/postgres-17-cluster/templates/object-store.yaml
|
|
apiVersion: barmancloud.cnpg.io/v1
|
|
kind: ObjectStore
|
|
metadata:
|
|
name: "immich-postgresql-17-recovery"
|
|
namespace: immich
|
|
labels:
|
|
helm.sh/chart: postgres-17-cluster-6.16.0
|
|
app.kubernetes.io/name: immich-postgresql-17
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/part-of: immich
|
|
app.kubernetes.io/version: "6.16.0"
|
|
app.kubernetes.io/managed-by: Helm
|
|
spec:
|
|
configuration:
|
|
destinationPath: s3://postgres-backups/cl01tl/immich/immich-postgresql-17-cluster
|
|
endpointURL: http://garage-main.garage:3900
|
|
wal:
|
|
compression: snappy
|
|
maxParallel: 1
|
|
data:
|
|
compression: snappy
|
|
jobs: 1
|
|
s3Credentials:
|
|
accessKeyId:
|
|
name: immich-postgresql-17-cluster-backup-secret-garage
|
|
key: ACCESS_KEY_ID
|
|
secretAccessKey:
|
|
name: immich-postgresql-17-cluster-backup-secret-garage
|
|
key: ACCESS_SECRET_KEY
|
|
---
|
|
# Source: immich/charts/postgres-17-cluster/templates/prometheus-rule.yaml
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: immich-postgresql-17-alert-rules
|
|
namespace: immich
|
|
labels:
|
|
helm.sh/chart: postgres-17-cluster-6.16.0
|
|
app.kubernetes.io/name: immich-postgresql-17
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/part-of: immich
|
|
app.kubernetes.io/version: "6.16.0"
|
|
app.kubernetes.io/managed-by: Helm
|
|
spec:
|
|
groups:
|
|
- name: cloudnative-pg/immich-postgresql-17
|
|
rules:
|
|
- alert: CNPGClusterBackendsWaitingWarning
|
|
annotations:
|
|
summary: CNPG Cluster a backend is waiting for longer than 5 minutes.
|
|
description: |-
|
|
Pod {{ $labels.pod }}
|
|
has been waiting for longer than 5 minutes
|
|
expr: |
|
|
cnpg_backends_waiting_total > 300
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
namespace: immich
|
|
cnpg_cluster: immich-postgresql-17-cluster
|
|
- alert: CNPGClusterDatabaseDeadlockConflictsWarning
|
|
annotations:
|
|
summary: CNPG Cluster has over 10 deadlock conflicts.
|
|
description: |-
|
|
There are over 10 deadlock conflicts in
|
|
{{ $labels.pod }}
|
|
expr: |
|
|
cnpg_pg_stat_database_deadlocks > 10
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
namespace: immich
|
|
cnpg_cluster: immich-postgresql-17-cluster
|
|
- alert: CNPGClusterHACritical
|
|
annotations:
|
|
summary: CNPG Cluster has no standby replicas!
|
|
description: |-
|
|
CloudNativePG Cluster "{{`{{`}} $labels.job {{`}}`}}" has no ready standby replicas. Your cluster at a severe
|
|
risk of data loss and downtime if the primary instance fails.
|
|
|
|
The primary instance is still online and able to serve queries, although connections to the `-ro` endpoint
|
|
will fail. The `-r` endpoint os operating at reduced capacity and all traffic is being served by the main.
|
|
|
|
This can happen during a normal fail-over or automated minor version upgrades in a cluster with 2 or less
|
|
instances. The replaced instance may need some time to catch-up with the cluster primary instance.
|
|
|
|
This alarm will be always trigger if your cluster is configured to run with only 1 instance. In this
|
|
case you may want to silence it.
|
|
runbook_url: https://github.com/cloudnative-pg/charts/blob/main/charts/cluster/docs/runbooks/CNPGClusterHACritical.md
|
|
expr: |
|
|
max by (job) (cnpg_pg_replication_streaming_replicas{namespace="immich"} - cnpg_pg_replication_is_wal_receiver_up{namespace="immich"}) < 1
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
namespace: immich
|
|
cnpg_cluster: immich-postgresql-17-cluster
|
|
- alert: CNPGClusterHAWarning
|
|
annotations:
|
|
summary: CNPG Cluster less than 2 standby replicas.
|
|
description: |-
|
|
CloudNativePG Cluster "{{`{{`}} $labels.job {{`}}`}}" has only {{`{{`}} $value {{`}}`}} standby replicas, putting
|
|
your cluster at risk if another instance fails. The cluster is still able to operate normally, although
|
|
the `-ro` and `-r` endpoints operate at reduced capacity.
|
|
|
|
This can happen during a normal fail-over or automated minor version upgrades. The replaced instance may
|
|
need some time to catch-up with the cluster primary instance.
|
|
|
|
This alarm will be constantly triggered if your cluster is configured to run with less than 3 instances.
|
|
In this case you may want to silence it.
|
|
runbook_url: https://github.com/cloudnative-pg/charts/blob/main/charts/cluster/docs/runbooks/CNPGClusterHAWarning.md
|
|
expr: |
|
|
max by (job) (cnpg_pg_replication_streaming_replicas{namespace="immich"} - cnpg_pg_replication_is_wal_receiver_up{namespace="immich"}) < 2
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
namespace: immich
|
|
cnpg_cluster: immich-postgresql-17-cluster
|
|
- alert: CNPGClusterHighConnectionsCritical
|
|
annotations:
|
|
summary: CNPG Instance maximum number of connections critical!
|
|
description: |-
|
|
CloudNativePG Cluster "immich/immich-postgresql-17-cluster" instance {{`{{`}} $labels.pod {{`}}`}} is using {{`{{`}} $value {{`}}`}}% of
|
|
the maximum number of connections.
|
|
runbook_url: https://github.com/cloudnative-pg/charts/blob/main/charts/cluster/docs/runbooks/CNPGClusterHighConnectionsCritical.md
|
|
expr: |
|
|
sum by (pod) (cnpg_backends_total{namespace="immich", pod=~"immich-postgresql-17-cluster-([1-9][0-9]*)$"}) / max by (pod) (cnpg_pg_settings_setting{name="max_connections", namespace="immich", pod=~"immich-postgresql-17-cluster-([1-9][0-9]*)$"}) * 100 > 95
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
namespace: immich
|
|
cnpg_cluster: immich-postgresql-17-cluster
|
|
- alert: CNPGClusterHighConnectionsWarning
|
|
annotations:
|
|
summary: CNPG Instance is approaching the maximum number of connections.
|
|
description: |-
|
|
CloudNativePG Cluster "immich/immich-postgresql-17-cluster" instance {{`{{`}} $labels.pod {{`}}`}} is using {{`{{`}} $value {{`}}`}}% of
|
|
the maximum number of connections.
|
|
runbook_url: https://github.com/cloudnative-pg/charts/blob/main/charts/cluster/docs/runbooks/CNPGClusterHighConnectionsWarning.md
|
|
expr: |
|
|
sum by (pod) (cnpg_backends_total{namespace="immich", pod=~"immich-postgresql-17-cluster-([1-9][0-9]*)$"}) / max by (pod) (cnpg_pg_settings_setting{name="max_connections", namespace="immich", pod=~"immich-postgresql-17-cluster-([1-9][0-9]*)$"}) * 100 > 80
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
namespace: immich
|
|
cnpg_cluster: immich-postgresql-17-cluster
|
|
- alert: CNPGClusterHighReplicationLag
|
|
annotations:
|
|
summary: CNPG Cluster high replication lag
|
|
description: |-
|
|
CloudNativePG Cluster "immich/immich-postgresql-17-cluster" is experiencing a high replication lag of
|
|
{{`{{`}} $value {{`}}`}}ms.
|
|
|
|
High replication lag indicates network issues, busy instances, slow queries or suboptimal configuration.
|
|
runbook_url: https://github.com/cloudnative-pg/charts/blob/main/charts/cluster/docs/runbooks/CNPGClusterHighReplicationLag.md
|
|
expr: |
|
|
max(cnpg_pg_replication_lag{namespace="immich",pod=~"immich-postgresql-17-cluster-([1-9][0-9]*)$"}) * 1000 > 1000
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
namespace: immich
|
|
cnpg_cluster: immich-postgresql-17-cluster
|
|
- alert: CNPGClusterInstancesOnSameNode
|
|
annotations:
|
|
summary: CNPG Cluster instances are located on the same node.
|
|
description: |-
|
|
CloudNativePG Cluster "immich/immich-postgresql-17-cluster" has {{`{{`}} $value {{`}}`}}
|
|
instances on the same node {{`{{`}} $labels.node {{`}}`}}.
|
|
|
|
A failure or scheduled downtime of a single node will lead to a potential service disruption and/or data loss.
|
|
runbook_url: https://github.com/cloudnative-pg/charts/blob/main/charts/cluster/docs/runbooks/CNPGClusterInstancesOnSameNode.md
|
|
expr: |
|
|
count by (node) (kube_pod_info{namespace="immich", pod=~"immich-postgresql-17-cluster-([1-9][0-9]*)$"}) > 1
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
namespace: immich
|
|
cnpg_cluster: immich-postgresql-17-cluster
|
|
- alert: CNPGClusterLongRunningTransactionWarning
|
|
annotations:
|
|
summary: CNPG Cluster query is taking longer than 5 minutes.
|
|
description: |-
|
|
CloudNativePG Cluster Pod {{ $labels.pod }}
|
|
is taking more than 5 minutes (300 seconds) for a query.
|
|
expr: |-
|
|
cnpg_backends_max_tx_duration_seconds > 300
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
namespace: immich
|
|
cnpg_cluster: immich-postgresql-17-cluster
|
|
- alert: CNPGClusterLowDiskSpaceCritical
|
|
annotations:
|
|
summary: CNPG Instance is running out of disk space!
|
|
description: |-
|
|
CloudNativePG Cluster "immich/immich-postgresql-17-cluster" is running extremely low on disk space. Check attached PVCs!
|
|
runbook_url: https://github.com/cloudnative-pg/charts/blob/main/charts/cluster/docs/runbooks/CNPGClusterLowDiskSpaceCritical.md
|
|
expr: |
|
|
max(max by(persistentvolumeclaim) (1 - kubelet_volume_stats_available_bytes{namespace="immich", persistentvolumeclaim=~"immich-postgresql-17-cluster-([1-9][0-9]*)$"} / kubelet_volume_stats_capacity_bytes{namespace="immich", persistentvolumeclaim=~"immich-postgresql-17-cluster-([1-9][0-9]*)$"})) > 0.9 OR
|
|
max(max by(persistentvolumeclaim) (1 - kubelet_volume_stats_available_bytes{namespace="immich", persistentvolumeclaim=~"immich-postgresql-17-cluster-([1-9][0-9]*)$-wal"} / kubelet_volume_stats_capacity_bytes{namespace="immich", persistentvolumeclaim=~"immich-postgresql-17-cluster-([1-9][0-9]*)$-wal"})) > 0.9 OR
|
|
max(sum by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace="immich", persistentvolumeclaim=~"immich-postgresql-17-cluster-([1-9][0-9]*)$-tbs.*"})
|
|
/
|
|
sum by (namespace,persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace="immich", persistentvolumeclaim=~"immich-postgresql-17-cluster-([1-9][0-9]*)$-tbs.*"})
|
|
*
|
|
on(namespace, persistentvolumeclaim) group_left(volume)
|
|
kube_pod_spec_volumes_persistentvolumeclaims_info{pod=~"immich-postgresql-17-cluster-([1-9][0-9]*)$"}
|
|
) > 0.9
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
namespace: immich
|
|
cnpg_cluster: immich-postgresql-17-cluster
|
|
- alert: CNPGClusterLowDiskSpaceWarning
|
|
annotations:
|
|
summary: CNPG Instance is running out of disk space.
|
|
description: |-
|
|
CloudNativePG Cluster "immich/immich-postgresql-17-cluster" is running low on disk space. Check attached PVCs.
|
|
runbook_url: https://github.com/cloudnative-pg/charts/blob/main/charts/cluster/docs/runbooks/CNPGClusterLowDiskSpaceWarning.md
|
|
expr: |
|
|
max(max by(persistentvolumeclaim) (1 - kubelet_volume_stats_available_bytes{namespace="immich", persistentvolumeclaim=~"immich-postgresql-17-cluster-([1-9][0-9]*)$"} / kubelet_volume_stats_capacity_bytes{namespace="immich", persistentvolumeclaim=~"immich-postgresql-17-cluster-([1-9][0-9]*)$"})) > 0.7 OR
|
|
max(max by(persistentvolumeclaim) (1 - kubelet_volume_stats_available_bytes{namespace="immich", persistentvolumeclaim=~"immich-postgresql-17-cluster-([1-9][0-9]*)$-wal"} / kubelet_volume_stats_capacity_bytes{namespace="immich", persistentvolumeclaim=~"immich-postgresql-17-cluster-([1-9][0-9]*)$-wal"})) > 0.7 OR
|
|
max(sum by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace="immich", persistentvolumeclaim=~"immich-postgresql-17-cluster-([1-9][0-9]*)$-tbs.*"})
|
|
/
|
|
sum by (namespace,persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace="immich", persistentvolumeclaim=~"immich-postgresql-17-cluster-([1-9][0-9]*)$-tbs.*"})
|
|
*
|
|
on(namespace, persistentvolumeclaim) group_left(volume)
|
|
kube_pod_spec_volumes_persistentvolumeclaims_info{pod=~"immich-postgresql-17-cluster-([1-9][0-9]*)$"}
|
|
) > 0.7
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
namespace: immich
|
|
cnpg_cluster: immich-postgresql-17-cluster
|
|
- alert: CNPGClusterOffline
|
|
annotations:
|
|
summary: CNPG Cluster has no running instances!
|
|
description: |-
|
|
CloudNativePG Cluster "immich/immich-postgresql-17-cluster" has no ready instances.
|
|
|
|
Having an offline cluster means your applications will not be able to access the database, leading to
|
|
potential service disruption and/or data loss.
|
|
runbook_url: https://github.com/cloudnative-pg/charts/blob/main/charts/cluster/docs/runbooks/CNPGClusterOffline.md
|
|
expr: |
|
|
(count(cnpg_collector_up{namespace="immich",pod=~"immich-postgresql-17-cluster-([1-9][0-9]*)$"}) OR on() vector(0)) == 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
namespace: immich
|
|
cnpg_cluster: immich-postgresql-17-cluster
|
|
- alert: CNPGClusterPGDatabaseXidAgeWarning
|
|
annotations:
|
|
summary: CNPG Cluster has a number of transactions from the frozen XID to the current one.
|
|
description: |-
|
|
Over 300,000,000 transactions from frozen xid
|
|
on pod {{ $labels.pod }}
|
|
expr: |
|
|
cnpg_pg_database_xid_age > 300000000
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
namespace: immich
|
|
cnpg_cluster: immich-postgresql-17-cluster
|
|
- alert: CNPGClusterPGReplicationWarning
|
|
annotations:
|
|
summary: CNPG Cluster standby is lagging behind the primary.
|
|
description: |-
|
|
Standby is lagging behind by over 300 seconds (5 minutes)
|
|
expr: |
|
|
cnpg_pg_replication_lag > 300
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
namespace: immich
|
|
cnpg_cluster: immich-postgresql-17-cluster
|
|
- alert: CNPGClusterReplicaFailingReplicationWarning
|
|
annotations:
|
|
summary: CNPG Cluster has a replica is failing to replicate.
|
|
description: |-
|
|
Replica {{ $labels.pod }}
|
|
is failing to replicate
|
|
expr: |
|
|
cnpg_pg_replication_in_recovery > cnpg_pg_replication_is_wal_receiver_up
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
namespace: immich
|
|
cnpg_cluster: immich-postgresql-17-cluster
|
|
- alert: CNPGClusterZoneSpreadWarning
|
|
annotations:
|
|
summary: CNPG Cluster instances in the same zone.
|
|
description: |-
|
|
CloudNativePG Cluster "immich/immich-postgresql-17-cluster" has instances in the same availability zone.
|
|
|
|
A disaster in one availability zone will lead to a potential service disruption and/or data loss.
|
|
runbook_url: https://github.com/cloudnative-pg/charts/blob/main/charts/cluster/docs/runbooks/CNPGClusterZoneSpreadWarning.md
|
|
expr: |
|
|
3 > count(count by (label_topology_kubernetes_io_zone) (kube_pod_info{namespace="immich", pod=~"immich-postgresql-17-cluster-([1-9][0-9]*)$"} * on(node,instance) group_left(label_topology_kubernetes_io_zone) kube_node_labels)) < 3
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
namespace: immich
|
|
cnpg_cluster: immich-postgresql-17-cluster
|
|
---
|
|
# Source: immich/templates/redis-replication.yaml
|
|
apiVersion: redis.redis.opstreelabs.in/v1beta2
|
|
kind: RedisReplication
|
|
metadata:
|
|
name: redis-replication-immich
|
|
namespace: immich
|
|
labels:
|
|
app.kubernetes.io/name: redis-replication-immich
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/part-of: immich
|
|
spec:
|
|
clusterSize: 3
|
|
podSecurityContext:
|
|
runAsUser: 1000
|
|
fsGroup: 1000
|
|
kubernetesConfig:
|
|
image: quay.io/opstree/redis:v8.0.3
|
|
imagePullPolicy: IfNotPresent
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 128Mi
|
|
storage:
|
|
volumeClaimTemplate:
|
|
spec:
|
|
storageClassName: ceph-block
|
|
accessModes: ["ReadWriteOnce"]
|
|
resources:
|
|
requests:
|
|
storage: 1Gi
|
|
redisExporter:
|
|
enabled: true
|
|
image: quay.io/opstree/redis-exporter:v1.48.0
|
|
---
|
|
# Source: immich/charts/postgres-17-cluster/templates/scheduled-backup.yaml
|
|
apiVersion: postgresql.cnpg.io/v1
|
|
kind: ScheduledBackup
|
|
metadata:
|
|
name: "immich-postgresql-17-daily-backup-scheduled-backup"
|
|
namespace: immich
|
|
labels:
|
|
helm.sh/chart: postgres-17-cluster-6.16.0
|
|
app.kubernetes.io/name: immich-postgresql-17
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/part-of: immich
|
|
app.kubernetes.io/version: "6.16.0"
|
|
app.kubernetes.io/managed-by: Helm
|
|
spec:
|
|
immediate: false
|
|
suspend: false
|
|
schedule: "0 0 0 * * *"
|
|
backupOwnerReference: self
|
|
cluster:
|
|
name: immich-postgresql-17-cluster
|
|
method: plugin
|
|
pluginConfiguration:
|
|
name: barman-cloud.cloudnative-pg.io
|
|
parameters:
|
|
barmanObjectName: "immich-postgresql-17-external-backup"
|
|
---
|
|
# Source: immich/charts/postgres-17-cluster/templates/scheduled-backup.yaml
|
|
apiVersion: postgresql.cnpg.io/v1
|
|
kind: ScheduledBackup
|
|
metadata:
|
|
name: "immich-postgresql-17-live-backup-scheduled-backup"
|
|
namespace: immich
|
|
labels:
|
|
helm.sh/chart: postgres-17-cluster-6.16.0
|
|
app.kubernetes.io/name: immich-postgresql-17
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/part-of: immich
|
|
app.kubernetes.io/version: "6.16.0"
|
|
app.kubernetes.io/managed-by: Helm
|
|
spec:
|
|
immediate: true
|
|
suspend: false
|
|
schedule: "0 0 0 * * *"
|
|
backupOwnerReference: self
|
|
cluster:
|
|
name: immich-postgresql-17-cluster
|
|
method: plugin
|
|
pluginConfiguration:
|
|
name: barman-cloud.cloudnative-pg.io
|
|
parameters:
|
|
barmanObjectName: "immich-postgresql-17-garage-local-backup"
|
|
---
|
|
# Source: immich/templates/service-monitor.yaml
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: ServiceMonitor
|
|
metadata:
|
|
name: immich
|
|
namespace: immich
|
|
labels:
|
|
app.kubernetes.io/name: immich
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/part-of: immich
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: immich
|
|
app.kubernetes.io/instance: immich
|
|
endpoints:
|
|
- port: metrics-api
|
|
interval: 3m
|
|
scrapeTimeout: 1m
|
|
path: /metrics
|
|
- port: metrics-ms
|
|
interval: 3m
|
|
scrapeTimeout: 1m
|
|
path: /metrics
|
|
---
|
|
# Source: immich/templates/service-monitor.yaml
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: ServiceMonitor
|
|
metadata:
|
|
name: redis-replication-immich
|
|
namespace: immich
|
|
labels:
|
|
app.kubernetes.io/name: redis-replication-immich
|
|
app.kubernetes.io/instance: immich
|
|
app.kubernetes.io/part-of: immich
|
|
redis-operator: "true"
|
|
env: production
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
redis_setup_type: replication
|
|
endpoints:
|
|
- port: redis-exporter
|
|
interval: 30s
|
|
scrapeTimeout: 10s
|