change to ceph filesystem, add backups

clusters/cl01tl/helm/vault/templates/config-map.yaml

@@ -0,0 +1,80 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vault-snapshot-script
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: vault-snapshot-script
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+data:
+  update.sh: |
+    MAX_RETRIES=5
+    SUCCESS=false
+
+    for ((i=1; i<=$MAX_RETRIES; i++)); do
+        if apk update --short &> /dev/null; then
+            echo ">> Attempt $i: Repositories are reachable";
+            SUCCESS=true;
+            break;
+        else
+            echo ">> Attempt $i: Connection failed, retrying in 5 seconds ...";
+            sleep 5;
+        fi;
+    done;
+
+    if [ "$SUCCESS" = false ]; then
+        echo ">> ERROR: Could not connect to apk repositories after $MAX_RETRIES attempts, exiting ...";
+        exit 1;
+    fi
+
+    if ! command -v jq 2>&1 >/dev/null;
+    then
+        echo "jq could not be found, installing";
+        apk add --no-cache jq;
+        if [ $? -eq 0 ]; then
+            echo ">> Installation successful";
+        else
+            echo ">> Installation failed with exit code $?";
+            exit 1;
+        fi;
+    fi;
+
+    echo " ";
+    echo ">> Fetching Vault token";
+    export VAULT_TOKEN=$(vault write auth/approle/login role_id=$VAULT_APPROLE_ROLE_ID secret_id=$VAULT_APPROLE_SECRET_ID -format=json | jq -r .auth.client_token);
+
+    echo " ";
+    echo ">> Taking Vault snapsot ...";
+    vault operator raft snapshot save /opt/backup/vault-snapshot-latest.snap;
+
+    echo " ";
+    echo ">> Archiving snapshot with $(date +"%Y%m%d-%H-%M") ...";
+    cp /opt/backup/vault-snapshot-latest.snap /opt/backup/vault-snapshot-$(date +"%Y%m%d-%H-%M").snap;
+
+    echo " ";
+    echo ">> Completed Vault snapshot"
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vault-snapshot-script
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: vault-snapshot-script
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+data:
+  update.sh: |
+    BACKUP_FOLDER=/opt/backup
+    BACKUP_FILE=$(ls -t $BACKUP_FOLDER | head -n 1)
+
+    echo " ";
+    echo ">> Running S3 backup for Vault snapshot";
+
+    if s3cmd put --no-check-md5 --no-check-certificate -v "$BACKUP_FOLDER/$BACKUP_FILE" "${BUCKET}/cl01tl/cl01tl-vault-snapshots/$BACKUP_FILE"; then
+        echo ">> Upload succeeded"
+    else
+        echo ">> ERROR: Upload failed"
+    fi

clusters/cl01tl/helm/vault/templates/external-secret.yaml

@@ -31,10 +31,70 @@ spec:
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
-  name: vault-s3cmd-config
+  name: vault-s3cmd-local-config
   namespace: {{ .Release.Namespace }}
   labels:
-    app.kubernetes.io/name: vault-s3cmd-config
+    app.kubernetes.io/name: vault-s3cmd-local-config
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: .s3cfg
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/vault-backups
+        metadataPolicy: None
+        property: s3cfg-local
+    - secretKey: BUCKET
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/vault-backups
+        metadataPolicy: None
+        property: BUCKET
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: vault-s3cmd-remote-config
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: vault-s3cmd-remote-config
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  data:
+    - secretKey: .s3cfg
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/vault-backups
+        metadataPolicy: None
+        property: s3cfg-remote
+    - secretKey: BUCKET
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        key: /garage/home-infra/vault-backups
+        metadataPolicy: None
+        property: BUCKET
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: vault-s3cmd-external-config
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: vault-s3cmd-external-config
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/part-of: {{ .Release.Name }}
 spec:

clusters/cl01tl/helm/vault/templates/persistent-volume-claim.yaml

@@ -15,3 +15,22 @@ spec:
   resources:
     requests:
       storage: 1Gi
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: vault-storage-backup
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: vault-storage-backup
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+spec:
+  volumeMode: Filesystem
+  storageClassName: ceph-filesystem
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Gi

clusters/cl01tl/helm/vault/values.yaml

@@ -35,9 +35,15 @@ vault:
       - name: vault-nfs-storage-backup
         persistentVolumeClaim:
           claimName: vault-nfs-storage-backup
+      - name: vault-nfs-storage-backup
+        persistentVolumeClaim:
+          claimName: vault-nfs-storage-backup
     volumeMounts:
-      - mountPath: /opt/backups/
+      - mountPath: /opt/backups-old/
         name: vault-nfs-storage-backup
+        readOnly: true
+      - mountPath: /opt/backups/
+        name: vault-storage-backup
         readOnly: false
     affinity: |
       podAntiAffinity:
@@ -176,26 +182,15 @@ snapshot:
             - /bin/ash
           args:
             - -ec
-            - |
+            - /scripts/snapshot.sh
-              apk add --no-cache jq;
-              echo ">> Running Vault snapshot"
-              export VAULT_TOKEN=$(vault write auth/approle/login role_id=$VAULT_APPROLE_ROLE_ID secret_id=$VAULT_APPROLE_SECRET_ID -format=json | jq -r .auth.client_token);
-              vault operator raft snapshot save /opt/backup/vault-snapshot-latest.snap;
-              cp /opt/backup/vault-snapshot-latest.snap /opt/backup/vault-snapshot-$(date +"%Y%m%d-%H-%M").snap;
-              cp /opt/backup/vault-snapshot-latest.snap /opt/backup/vault-snapshot-s3.snap;
-              echo ">> Completed Vault snapshot"
           envFrom:
             - secretRef:
                 name: vault-snapshot-agent-token
           env:
             - name: VAULT_ADDR
               value: http://vault-active.vault.svc.cluster.local:8200
-          resources:
-            requests:
-              cpu: 10m
-              memory: 64Mi
       containers:
-        s3-backup:
+        s3-backup-local:
           image:
             repository: d3fk/s3cmd
             tag: latest@sha256:ed348a0fae5723d2e62636c175baf4dfaf732a790179ca675d1f24f863d0d68f
@@ -204,43 +199,121 @@ snapshot:
             - /bin/sh
           args:
             - -ec
-            - |
+            - /scripts/backup.sh
-              echo ">> Running S3 backup for Vault snapshot"
-              s3cmd put --no-check-md5 --no-check-certificate -v /opt/backup/vault-snapshot-s3.snap ${BUCKET}/cl01tl/cl01tl-vault-snapshots/vault-snapshot-$(date +"%Y%m%d-%H-%M").snap;
-              rm -f /opt/backup/vault-snapshot-s3.snap;
-              echo ">> Completed S3 backup for Vault snapshot"
           env:
             - name: BUCKET
               valueFrom:
                 secretKeyRef:
-                  name: vault-s3cmd-config
+                  name: vault-s3cmd-local-config
+                  key: BUCKET
+        s3-backup-remote:
+          image:
+            repository: d3fk/s3cmd
+            tag: latest@sha256:ed348a0fae5723d2e62636c175baf4dfaf732a790179ca675d1f24f863d0d68f
+            pullPolicy: IfNotPresent
+          command:
+            - /bin/sh
+          args:
+            - -ec
+            - /scripts/backup.sh
+          env:
+            - name: BUCKET
+              valueFrom:
+                secretKeyRef:
+                  name: vault-s3cmd-remote-config
+                  key: BUCKET
+        s3-backup-external:
+          image:
+            repository: d3fk/s3cmd
+            tag: latest@sha256:ed348a0fae5723d2e62636c175baf4dfaf732a790179ca675d1f24f863d0d68f
+            pullPolicy: IfNotPresent
+          command:
+            - /bin/sh
+          args:
+            - -ec
+            - /scripts/backup.sh
+          env:
+            - name: BUCKET
+              valueFrom:
+                secretKeyRef:
+                  name: vault-s3cmd-external-config
                   key: BUCKET
-          resources:
-            requests:
-              cpu: 100m
-              memory: 128Mi
   persistence:
-    config:
+    snapshot-script:
-      existingClaim: vault-nfs-storage-backup
+      enabled: true
+      type: configMap
+      name: vault-snapshot-script
+      defaultMode: 0755
+      advancedMounts:
+        snapshot:
+          snapshot:
+            - path: /scripts/snapshot.sh
+              subPath: snapshot.sh
+    backup-script:
+      enabled: true
+      type: configMap
+      name: vault-backup-script
+      defaultMode: 0755
+      advancedMounts:
+        snapshot:
+          s3-backup-local:
+            - path: /scripts/backup.sh
+              subPath: backup.sh
+          s3-backup-remote:
+            - path: /scripts/backup.sh
+              subPath: backup.sh
+          s3-backup-external:
+            - path: /scripts/backup.sh
+              subPath: backup.sh
+    s3cmd-local-config:
+      enabled: true
+      type: secret
+      name: vault-s3cmd-local-config
+      advancedMounts:
+        snapshot:
+          s3-backup-local:
+            - path: /root/.s3cfg
+              readOnly: true
+              mountPropagation: None
+              subPath: .s3cfg
+    s3cmd-remote-config:
+      enabled: true
+      type: secret
+      name: vault-s3cmd-remote-config
+      advancedMounts:
+        snapshot:
+          s3-backup-remote:
+            - path: /root/.s3cfg
+              readOnly: true
+              mountPropagation: None
+              subPath: .s3cfg
+    s3cmd-external-config:
+      enabled: true
+      type: secret
+      name: vault-s3cmd-external-config
+      advancedMounts:
+        snapshot:
+          s3-backup-external:
+            - path: /root/.s3cfg
+              readOnly: true
+              mountPropagation: None
+              subPath: .s3cfg
+    backup:
+      existingClaim: vault-storage-backup
       advancedMounts:
         snapshot:
           snapshot:
             - path: /opt/backup
               readOnly: false
-          s3-backup:
+          s3-backup-local:
+            - path: /opt/backup
+              readOnly: false
+          s3-backup-remote:
+            - path: /opt/backup
+              readOnly: false
+          s3-backup-external:
             - path: /opt/backup
               readOnly: false
-    s3cmd-config:
-      enabled: true
-      type: secret
-      name: vault-s3cmd-config
-      advancedMounts:
-        snapshot:
-          s3-backup:
-            - path: /root/.s3cfg
-              readOnly: true
-              mountPropagation: None
-              subPath: .s3cfg
 unseal:
   global:
     fullnameOverride: vault-unseal