apiVersion: v1 kind: ConfigMap metadata: name: talos-prune-script namespace: talos labels: app.kubernetes.io/name: talos-prune-script app.kubernetes.io/instance: talos app.kubernetes.io/part-of: talos data: prune.sh: | DATE_RANGE=$(date -d @$(( $(date +%s) - $DATE_RANGE_SECONDS )) +%Y-%m-%dT%H:%M:%SZ); FILE_MATCH="${BUCKET}/cl01tl/etcd/cl01tl-${DATE_RANGE}.snap.age"; ERROR=false; echo " "; echo ">> Running S3 prune for Talos backup repository ${TARGET} ..."; echo " "; echo ">> Configured Date Range is $(date -u -d @${DATE_RANGE_SECONDS} +"%j days, %H hours, %M minutes")"; echo ">> Backups prior to '$DATE_RANGE' will be removed"; FILES=$(s3cmd ls --no-check-certificate ${BUCKET}/cl01tl/etcd/ | awk -v file_match="$FILE_MATCH" '$4 < file_match {print $4}'); if [ $? -ne 0 ]; then ERROR=true; echo " "; echo ">> Detected error, will send message to ntfy"; elif [ -n "${FILES}" ]; then echo " "; echo ">> Backups to be removed:"; echo "$FILES" echo " "; echo ">> Deleting ..."; $FILES | while read file; do s3cmd del --no-check-certificate -v "$file"; if [ $? -ne 0 ]; then ERROR=true; echo ">> Detected error, will send message to ntfy"; fi; done; else echo " "; echo ">> No backups to remove"; fi; if [ "$ERROR" = "true" ]; then MAX_RETRIES=5; SUCCESS=false; echo " "; echo ">> Sending message to ntfy using curl ..."; echo " "; echo ">> Verifying required commands ..."; for i in $(seq 1 "$MAX_RETRIES"); do if apk update 2>&1 >/dev/null; then echo ">> Attempt $i: Repositories are reachable"; SUCCESS=true; break; else echo ">> Attempt $i: Connection failed, retrying in 5 seconds ..."; sleep 5; fi; done; if [ "$SUCCESS" = false ]; then echo ">> ERROR: Could not connect to apk repositories after $MAX_RETRIES attempts, exiting ..."; exit 1; fi if ! command -v curl 2>&1 >/dev/null; then echo ">> Command curl could not be found, installing"; apk add --no-cache -q curl; if [ $? -eq 0 ]; then echo ">> Installation successful"; else echo ">> Installation failed with exit code $?"; exit 1; fi; fi; echo " "; echo ">> Sending to NTFY ..."; HTTP_STATUS=$(curl \ --silent \ --write-out '%{http_code}' \ -H "Authorization: Bearer ${NTFY_TOKEN}" \ -H "X-Priority: 5" \ -H "X-Tags: warning" \ -H "X-Title: Talos Backup Failed for ${TARGET}" \ -d "$MESSAGE" \ ${NTFY_ENDPOINT}/${NTFY_TOPIC} ); echo ">> HTTP Status Code: $HTTP_STATUS"; fi; echo " "; echo ">> Completed S3 prune for Talos backup repository ${TARGET}"; --- apiVersion: batch/v1 kind: CronJob metadata: name: etcd-defrag-defrag-1 labels: app.kubernetes.io/controller: defrag-1 app.kubernetes.io/instance: talos app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: talos helm.sh/chart: etcd-defrag-4.6.2 namespace: talos spec: suspend: false concurrencyPolicy: Forbid startingDeadlineSeconds: 90 timeZone: US/Central schedule: "0 0 * * 0" successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 1 jobTemplate: spec: parallelism: 1 backoffLimit: 3 template: metadata: labels: app.kubernetes.io/controller: defrag-1 app.kubernetes.io/instance: talos app.kubernetes.io/name: talos spec: enableServiceLinks: false serviceAccountName: default automountServiceAccountToken: true hostIPC: false hostNetwork: false hostPID: false dnsPolicy: ClusterFirst restartPolicy: Never nodeSelector: node-role.kubernetes.io/control-plane: "" tolerations: - effect: NoSchedule key: node-role.kubernetes.io/control-plane operator: Exists containers: - args: - etcd - defrag - -n - 10.232.1.11 env: - name: TALOSCONFIG value: /tmp/.talos/config image: ghcr.io/siderolabs/talosctl:v1.12.5 imagePullPolicy: IfNotPresent name: main volumeMounts: - mountPath: /tmp/.talos/config mountPropagation: None name: talos-config-1 readOnly: true subPath: config volumes: - name: talos-config-1 secret: secretName: talos-etcd-defrag-secret --- apiVersion: batch/v1 kind: CronJob metadata: name: etcd-defrag-defrag-2 labels: app.kubernetes.io/controller: defrag-2 app.kubernetes.io/instance: talos app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: talos helm.sh/chart: etcd-defrag-4.6.2 namespace: talos spec: suspend: false concurrencyPolicy: Forbid startingDeadlineSeconds: 90 timeZone: US/Central schedule: "10 0 * * 0" successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 1 jobTemplate: spec: parallelism: 1 backoffLimit: 3 template: metadata: labels: app.kubernetes.io/controller: defrag-2 app.kubernetes.io/instance: talos app.kubernetes.io/name: talos spec: enableServiceLinks: false serviceAccountName: default automountServiceAccountToken: true hostIPC: false hostNetwork: false hostPID: false dnsPolicy: ClusterFirst restartPolicy: Never nodeSelector: node-role.kubernetes.io/control-plane: "" tolerations: - effect: NoSchedule key: node-role.kubernetes.io/control-plane operator: Exists containers: - args: - etcd - defrag - -n - 10.232.1.12 env: - name: TALOSCONFIG value: /tmp/.talos/config image: ghcr.io/siderolabs/talosctl:v1.12.5 imagePullPolicy: IfNotPresent name: main volumeMounts: - mountPath: /tmp/.talos/config mountPropagation: None name: talos-config-2 readOnly: true subPath: config volumes: - name: talos-config-2 secret: secretName: talos-etcd-defrag-secret --- apiVersion: batch/v1 kind: CronJob metadata: name: etcd-defrag-defrag-3 labels: app.kubernetes.io/controller: defrag-3 app.kubernetes.io/instance: talos app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: talos helm.sh/chart: etcd-defrag-4.6.2 namespace: talos spec: suspend: false concurrencyPolicy: Forbid startingDeadlineSeconds: 90 timeZone: US/Central schedule: "20 0 * * 0" successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 1 jobTemplate: spec: parallelism: 1 backoffLimit: 3 template: metadata: labels: app.kubernetes.io/controller: defrag-3 app.kubernetes.io/instance: talos app.kubernetes.io/name: talos spec: enableServiceLinks: false serviceAccountName: default automountServiceAccountToken: true hostIPC: false hostNetwork: false hostPID: false dnsPolicy: ClusterFirst restartPolicy: Never nodeSelector: node-role.kubernetes.io/control-plane: "" tolerations: - effect: NoSchedule key: node-role.kubernetes.io/control-plane operator: Exists containers: - args: - etcd - defrag - -n - 10.232.1.13 env: - name: TALOSCONFIG value: /tmp/.talos/config image: ghcr.io/siderolabs/talosctl:v1.12.5 imagePullPolicy: IfNotPresent name: main volumeMounts: - mountPath: /tmp/.talos/config mountPropagation: None name: talos-config-3 readOnly: true subPath: config volumes: - name: talos-config-3 secret: secretName: talos-etcd-defrag-secret --- apiVersion: batch/v1 kind: CronJob metadata: name: talos-external labels: app.kubernetes.io/controller: external app.kubernetes.io/instance: talos app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: talos helm.sh/chart: etcd-backup-4.6.2 namespace: talos spec: suspend: false concurrencyPolicy: Forbid startingDeadlineSeconds: 90 timeZone: US/Central schedule: "0 4 * * *" successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 1 jobTemplate: spec: parallelism: 1 backoffLimit: 3 template: metadata: labels: app.kubernetes.io/controller: external app.kubernetes.io/instance: talos app.kubernetes.io/name: talos spec: enableServiceLinks: false serviceAccountName: default automountServiceAccountToken: true hostIPC: false hostNetwork: false hostPID: false dnsPolicy: ClusterFirst restartPolicy: Never nodeSelector: node-role.kubernetes.io/control-plane: "" tolerations: - effect: NoSchedule key: node-role.kubernetes.io/control-plane operator: Exists containers: - command: - /talos-backup env: - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: key: AWS_ACCESS_KEY_ID name: talos-etcd-backup-external-secret - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: key: AWS_SECRET_ACCESS_KEY name: talos-etcd-backup-external-secret - name: AWS_REGION value: nyc3 - name: CUSTOM_S3_ENDPOINT value: https://nyc3.digitaloceanspaces.com - name: BUCKET value: talos-backups-bee8585f7b8a4d0239c9b823 - name: S3_PREFIX value: cl01tl/etcd - name: CLUSTER_NAME value: cl01tl - name: AGE_X25519_PUBLIC_KEY valueFrom: secretKeyRef: key: AGE_X25519_PUBLIC_KEY name: talos-etcd-backup-external-secret - name: USE_PATH_STYLE value: "false" image: ghcr.io/siderolabs/talos-backup:v0.1.0-beta.3-5-g07d09ec@sha256:96054af026b6255ec14d198f2f10ad6c813b335a2e21a76804365c053dd4ba7b imagePullPolicy: IfNotPresent name: backup securityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL runAsGroup: 1000 runAsNonRoot: true runAsUser: 1000 seccompProfile: type: RuntimeDefault volumeMounts: - mountPath: /var/run/secrets/talos.dev mountPropagation: None name: secret readOnly: true - mountPath: /.talos name: talos-external - mountPath: /tmp name: tmp-external workingDir: /tmp - args: - -ec - /scripts/prune.sh command: - /bin/sh env: - name: TARGET value: External - name: DATE_RANGE_SECONDS value: "1209600" envFrom: - secretRef: name: talos-etcd-backup-external-secret - secretRef: name: talos-backup-ntfy-secret image: d3fk/s3cmd:latest@sha256:a41234c2b43d6cfa0d51c9523a2d7925f7f21297a41d69932946c3e364d32b5e imagePullPolicy: IfNotPresent name: s3-prune volumeMounts: - mountPath: /scripts/prune.sh name: prune-script subPath: prune.sh - mountPath: /root/.s3cfg mountPropagation: None name: s3cmd-config-external readOnly: true subPath: .s3cfg volumes: - configMap: defaultMode: 493 name: talos-prune-script name: prune-script - name: s3cmd-config-external secret: secretName: talos-etcd-backup-external-secret - name: secret secret: secretName: talos-backup-secrets - emptyDir: medium: Memory name: talos-external - emptyDir: medium: Memory name: tmp-external --- apiVersion: batch/v1 kind: CronJob metadata: name: talos-local labels: app.kubernetes.io/controller: local app.kubernetes.io/instance: talos app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: talos helm.sh/chart: etcd-backup-4.6.2 namespace: talos spec: suspend: false concurrencyPolicy: Forbid startingDeadlineSeconds: 90 timeZone: US/Central schedule: "0 2 * * *" successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 1 jobTemplate: spec: parallelism: 1 backoffLimit: 3 template: metadata: labels: app.kubernetes.io/controller: local app.kubernetes.io/instance: talos app.kubernetes.io/name: talos spec: enableServiceLinks: false serviceAccountName: default automountServiceAccountToken: true hostIPC: false hostNetwork: false hostPID: false dnsPolicy: ClusterFirst restartPolicy: Never nodeSelector: node-role.kubernetes.io/control-plane: "" tolerations: - effect: NoSchedule key: node-role.kubernetes.io/control-plane operator: Exists containers: - command: - /talos-backup env: - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: key: AWS_ACCESS_KEY_ID name: talos-etcd-backup-local-secret - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: key: AWS_SECRET_ACCESS_KEY name: talos-etcd-backup-local-secret - name: AWS_REGION value: us-east-1 - name: CUSTOM_S3_ENDPOINT value: http://garage-main.garage:3900 - name: BUCKET value: talos-backups - name: S3_PREFIX value: cl01tl/etcd - name: CLUSTER_NAME value: cl01tl - name: AGE_X25519_PUBLIC_KEY valueFrom: secretKeyRef: key: AGE_X25519_PUBLIC_KEY name: talos-etcd-backup-local-secret - name: USE_PATH_STYLE value: "false" image: ghcr.io/siderolabs/talos-backup:v0.1.0-beta.3@sha256:05c86663b251a407551dc948097e32e163a345818117eb52c573b0447bd0c7a7 imagePullPolicy: IfNotPresent name: backup securityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL runAsGroup: 1000 runAsNonRoot: true runAsUser: 1000 seccompProfile: type: RuntimeDefault volumeMounts: - mountPath: /var/run/secrets/talos.dev mountPropagation: None name: secret readOnly: true - mountPath: /.talos name: talos-local - mountPath: /tmp name: tmp-local workingDir: /tmp - args: - -ec - /scripts/prune.sh command: - /bin/sh env: - name: TARGET value: Local - name: DATE_RANGE_SECONDS value: "2419200" envFrom: - secretRef: name: talos-etcd-backup-local-secret - secretRef: name: talos-backup-ntfy-secret image: d3fk/s3cmd:latest@sha256:a41234c2b43d6cfa0d51c9523a2d7925f7f21297a41d69932946c3e364d32b5e imagePullPolicy: IfNotPresent name: s3-prune volumeMounts: - mountPath: /scripts/prune.sh name: prune-script subPath: prune.sh - mountPath: /root/.s3cfg mountPropagation: None name: s3cmd-config-local readOnly: true subPath: .s3cfg volumes: - configMap: defaultMode: 493 name: talos-prune-script name: prune-script - name: s3cmd-config-local secret: secretName: talos-etcd-backup-local-secret - name: secret secret: secretName: talos-backup-secrets - emptyDir: medium: Memory name: talos-local - emptyDir: medium: Memory name: tmp-local --- apiVersion: batch/v1 kind: CronJob metadata: name: talos-remote labels: app.kubernetes.io/controller: remote app.kubernetes.io/instance: talos app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: talos helm.sh/chart: etcd-backup-4.6.2 namespace: talos spec: suspend: true concurrencyPolicy: Forbid startingDeadlineSeconds: 90 timeZone: US/Central schedule: "0 3 * * *" successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 1 jobTemplate: spec: parallelism: 1 backoffLimit: 3 template: metadata: labels: app.kubernetes.io/controller: remote app.kubernetes.io/instance: talos app.kubernetes.io/name: talos spec: enableServiceLinks: false serviceAccountName: default automountServiceAccountToken: true hostIPC: false hostNetwork: false hostPID: false dnsPolicy: ClusterFirst restartPolicy: Never nodeSelector: node-role.kubernetes.io/control-plane: "" tolerations: - effect: NoSchedule key: node-role.kubernetes.io/control-plane operator: Exists containers: - command: - /talos-backup env: - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: key: AWS_ACCESS_KEY_ID name: talos-etcd-backup-remote-secret - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: key: AWS_SECRET_ACCESS_KEY name: talos-etcd-backup-remote-secret - name: AWS_REGION value: us-east-1 - name: CUSTOM_S3_ENDPOINT value: https://garage-ps10rp.boreal-beaufort.ts.net:3900 - name: BUCKET value: talos-backups - name: S3_PREFIX value: cl01tl/etcd - name: CLUSTER_NAME value: cl01tl - name: AGE_X25519_PUBLIC_KEY valueFrom: secretKeyRef: key: AGE_X25519_PUBLIC_KEY name: talos-etcd-backup-remote-secret - name: USE_PATH_STYLE value: "false" image: ghcr.io/siderolabs/talos-backup:v0.1.0-beta.3@sha256:05c86663b251a407551dc948097e32e163a345818117eb52c573b0447bd0c7a7 imagePullPolicy: IfNotPresent name: backup securityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL runAsGroup: 1000 runAsNonRoot: true runAsUser: 1000 seccompProfile: type: RuntimeDefault volumeMounts: - mountPath: /var/run/secrets/talos.dev mountPropagation: None name: secret readOnly: true - mountPath: /.talos name: talos-remote - mountPath: /tmp name: tmp-remote workingDir: /tmp - args: - -ec - /scripts/prune.sh command: - /bin/sh env: - name: TARGET value: Remote - name: DATE_RANGE_SECONDS value: "2419200" envFrom: - secretRef: name: talos-etcd-backup-remote-secret - secretRef: name: talos-backup-ntfy-secret image: d3fk/s3cmd:latest@sha256:a41234c2b43d6cfa0d51c9523a2d7925f7f21297a41d69932946c3e364d32b5e imagePullPolicy: IfNotPresent name: s3-prune volumeMounts: - mountPath: /scripts/prune.sh name: prune-script subPath: prune.sh - mountPath: /root/.s3cfg mountPropagation: None name: s3cmd-config-remote readOnly: true subPath: .s3cfg volumes: - configMap: defaultMode: 493 name: talos-prune-script name: prune-script - name: s3cmd-config-remote secret: secretName: talos-etcd-backup-remote-secret - name: secret secret: secretName: talos-backup-secrets - emptyDir: medium: Memory name: talos-remote - emptyDir: medium: Memory name: tmp-remote --- apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: name: talos-backup-ntfy-secret namespace: talos labels: app.kubernetes.io/name: talos-backup-ntfy-secret app.kubernetes.io/instance: talos app.kubernetes.io/part-of: talos spec: secretStoreRef: kind: ClusterSecretStore name: vault data: - secretKey: NTFY_TOKEN remoteRef: conversionStrategy: Default decodingStrategy: None key: /ntfy/user/cl01tl metadataPolicy: None property: token - secretKey: NTFY_ENDPOINT remoteRef: conversionStrategy: Default decodingStrategy: None key: /ntfy/user/cl01tl metadataPolicy: None property: endpoint - secretKey: NTFY_TOPIC remoteRef: conversionStrategy: Default decodingStrategy: None key: /cl01tl/talos/etcd-backup metadataPolicy: None property: NTFY_TOPIC --- apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: name: talos-etcd-backup-external-secret namespace: talos labels: app.kubernetes.io/name: talos-etcd-backup-external-secret app.kubernetes.io/instance: talos app.kubernetes.io/part-of: talos spec: secretStoreRef: kind: ClusterSecretStore name: vault data: - secretKey: AWS_ACCESS_KEY_ID remoteRef: conversionStrategy: Default decodingStrategy: None key: /digital-ocean/home-infra/etcd-backup metadataPolicy: None property: AWS_ACCESS_KEY_ID - secretKey: AWS_SECRET_ACCESS_KEY remoteRef: conversionStrategy: Default decodingStrategy: None key: /digital-ocean/home-infra/etcd-backup metadataPolicy: None property: AWS_SECRET_ACCESS_KEY - secretKey: .s3cfg remoteRef: conversionStrategy: Default decodingStrategy: None key: /digital-ocean/home-infra/etcd-backup metadataPolicy: None property: s3cfg - secretKey: BUCKET remoteRef: conversionStrategy: Default decodingStrategy: None key: /digital-ocean/home-infra/etcd-backup metadataPolicy: None property: BUCKET - secretKey: AGE_X25519_PUBLIC_KEY remoteRef: conversionStrategy: Default decodingStrategy: None key: /cl01tl/talos/etcd-backup metadataPolicy: None property: AGE_X25519_PUBLIC_KEY --- apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: name: talos-etcd-backup-local-secret namespace: talos labels: app.kubernetes.io/name: talos-etcd-backup-local-secret app.kubernetes.io/instance: talos app.kubernetes.io/part-of: talos spec: secretStoreRef: kind: ClusterSecretStore name: vault data: - secretKey: AWS_ACCESS_KEY_ID remoteRef: conversionStrategy: Default decodingStrategy: None key: /garage/home-infra/talos-backups metadataPolicy: None property: ACCESS_KEY_ID - secretKey: AWS_SECRET_ACCESS_KEY remoteRef: conversionStrategy: Default decodingStrategy: None key: /garage/home-infra/talos-backups metadataPolicy: None property: ACCESS_SECRET_KEY - secretKey: .s3cfg remoteRef: conversionStrategy: Default decodingStrategy: None key: /garage/home-infra/talos-backups metadataPolicy: None property: s3cfg-local - secretKey: BUCKET remoteRef: conversionStrategy: Default decodingStrategy: None key: /garage/home-infra/talos-backups metadataPolicy: None property: BUCKET - secretKey: AGE_X25519_PUBLIC_KEY remoteRef: conversionStrategy: Default decodingStrategy: None key: /cl01tl/talos/etcd-backup metadataPolicy: None property: AGE_X25519_PUBLIC_KEY --- apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: name: talos-etcd-backup-remote-secret namespace: talos labels: app.kubernetes.io/name: talos-etcd-backup-remote-secret app.kubernetes.io/instance: talos app.kubernetes.io/part-of: talos spec: secretStoreRef: kind: ClusterSecretStore name: vault data: - secretKey: AWS_ACCESS_KEY_ID remoteRef: conversionStrategy: Default decodingStrategy: None key: /garage/home-infra/talos-backups metadataPolicy: None property: ACCESS_KEY_ID - secretKey: AWS_SECRET_ACCESS_KEY remoteRef: conversionStrategy: Default decodingStrategy: None key: /garage/home-infra/talos-backups metadataPolicy: None property: ACCESS_SECRET_KEY - secretKey: .s3cfg remoteRef: conversionStrategy: Default decodingStrategy: None key: /garage/home-infra/talos-backups metadataPolicy: None property: s3cfg-remote - secretKey: BUCKET remoteRef: conversionStrategy: Default decodingStrategy: None key: /garage/home-infra/talos-backups metadataPolicy: None property: BUCKET - secretKey: AGE_X25519_PUBLIC_KEY remoteRef: conversionStrategy: Default decodingStrategy: None key: /cl01tl/talos/etcd-backup metadataPolicy: None property: AGE_X25519_PUBLIC_KEY --- apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: name: talos-etcd-defrag-secret namespace: talos labels: app.kubernetes.io/name: talos-etcd-defrag-secret app.kubernetes.io/instance: talos app.kubernetes.io/part-of: talos spec: secretStoreRef: kind: ClusterSecretStore name: vault data: - secretKey: config remoteRef: conversionStrategy: Default decodingStrategy: None key: /cl01tl/talos/etcd-defrag metadataPolicy: None property: config --- apiVersion: v1 kind: Secret metadata: name: talos-backup-secrets namespace: talos labels: app.kubernetes.io/name: talos-backup-secrets app.kubernetes.io/instance: talos app.kubernetes.io/part-of: talos annotations: kubernetes.io/service-account.name: talos-backup-secrets --- apiVersion: talos.dev/v1alpha1 kind: ServiceAccount metadata: name: talos-backup-secrets namespace: talos labels: app.kubernetes.io/name: talos-backup-secrets app.kubernetes.io/instance: talos app.kubernetes.io/part-of: talos spec: roles: - os:etcd:backup