diff --git a/clusters/cl01tl/helm/vault/templates/config-map.yaml b/clusters/cl01tl/helm/vault/templates/config-map.yaml new file mode 100644 index 000000000..0a3c33446 --- /dev/null +++ b/clusters/cl01tl/helm/vault/templates/config-map.yaml @@ -0,0 +1,80 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: vault-snapshot-script + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: vault-snapshot-script + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/part-of: {{ .Release.Name }} +data: + update.sh: | + MAX_RETRIES=5 + SUCCESS=false + + for ((i=1; i<=$MAX_RETRIES; i++)); do + if apk update --short &> /dev/null; then + echo ">> Attempt $i: Repositories are reachable"; + SUCCESS=true; + break; + else + echo ">> Attempt $i: Connection failed, retrying in 5 seconds ..."; + sleep 5; + fi; + done; + + if [ "$SUCCESS" = false ]; then + echo ">> ERROR: Could not connect to apk repositories after $MAX_RETRIES attempts, exiting ..."; + exit 1; + fi + + if ! command -v jq 2>&1 >/dev/null; + then + echo "jq could not be found, installing"; + apk add --no-cache jq; + if [ $? -eq 0 ]; then + echo ">> Installation successful"; + else + echo ">> Installation failed with exit code $?"; + exit 1; + fi; + fi; + + echo " "; + echo ">> Fetching Vault token"; + export VAULT_TOKEN=$(vault write auth/approle/login role_id=$VAULT_APPROLE_ROLE_ID secret_id=$VAULT_APPROLE_SECRET_ID -format=json | jq -r .auth.client_token); + + echo " "; + echo ">> Taking Vault snapsot ..."; + vault operator raft snapshot save /opt/backup/vault-snapshot-latest.snap; + + echo " "; + echo ">> Archiving snapshot with $(date +"%Y%m%d-%H-%M") ..."; + cp /opt/backup/vault-snapshot-latest.snap /opt/backup/vault-snapshot-$(date +"%Y%m%d-%H-%M").snap; + + echo " "; + echo ">> Completed Vault snapshot" + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vault-snapshot-script + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: vault-snapshot-script + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/part-of: {{ .Release.Name }} +data: + update.sh: | + BACKUP_FOLDER=/opt/backup + BACKUP_FILE=$(ls -t $BACKUP_FOLDER | head -n 1) + + echo " "; + echo ">> Running S3 backup for Vault snapshot"; + + if s3cmd put --no-check-md5 --no-check-certificate -v "$BACKUP_FOLDER/$BACKUP_FILE" "${BUCKET}/cl01tl/cl01tl-vault-snapshots/$BACKUP_FILE"; then + echo ">> Upload succeeded" + else + echo ">> ERROR: Upload failed" + fi diff --git a/clusters/cl01tl/helm/vault/templates/external-secret.yaml b/clusters/cl01tl/helm/vault/templates/external-secret.yaml index 5b90c8646..5d1453b53 100644 --- a/clusters/cl01tl/helm/vault/templates/external-secret.yaml +++ b/clusters/cl01tl/helm/vault/templates/external-secret.yaml @@ -31,10 +31,70 @@ spec: apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: - name: vault-s3cmd-config + name: vault-s3cmd-local-config namespace: {{ .Release.Namespace }} labels: - app.kubernetes.io/name: vault-s3cmd-config + app.kubernetes.io/name: vault-s3cmd-local-config + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/part-of: {{ .Release.Name }} +spec: + secretStoreRef: + kind: ClusterSecretStore + name: vault + data: + - secretKey: .s3cfg + remoteRef: + conversionStrategy: Default + decodingStrategy: None + key: /garage/home-infra/vault-backups + metadataPolicy: None + property: s3cfg-local + - secretKey: BUCKET + remoteRef: + conversionStrategy: Default + decodingStrategy: None + key: /garage/home-infra/vault-backups + metadataPolicy: None + property: BUCKET + +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: vault-s3cmd-remote-config + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: vault-s3cmd-remote-config + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/part-of: {{ .Release.Name }} +spec: + secretStoreRef: + kind: ClusterSecretStore + name: vault + data: + - secretKey: .s3cfg + remoteRef: + conversionStrategy: Default + decodingStrategy: None + key: /garage/home-infra/vault-backups + metadataPolicy: None + property: s3cfg-remote + - secretKey: BUCKET + remoteRef: + conversionStrategy: Default + decodingStrategy: None + key: /garage/home-infra/vault-backups + metadataPolicy: None + property: BUCKET + +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: vault-s3cmd-external-config + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: vault-s3cmd-external-config app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/part-of: {{ .Release.Name }} spec: diff --git a/clusters/cl01tl/helm/vault/templates/persistent-volume-claim.yaml b/clusters/cl01tl/helm/vault/templates/persistent-volume-claim.yaml index a3d4eb322..04d2d1ca0 100644 --- a/clusters/cl01tl/helm/vault/templates/persistent-volume-claim.yaml +++ b/clusters/cl01tl/helm/vault/templates/persistent-volume-claim.yaml @@ -15,3 +15,22 @@ spec: resources: requests: storage: 1Gi + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: vault-storage-backup + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: vault-storage-backup + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/part-of: {{ .Release.Name }} +spec: + volumeMode: Filesystem + storageClassName: ceph-filesystem + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi diff --git a/clusters/cl01tl/helm/vault/values.yaml b/clusters/cl01tl/helm/vault/values.yaml index d58c629f1..c72adf056 100644 --- a/clusters/cl01tl/helm/vault/values.yaml +++ b/clusters/cl01tl/helm/vault/values.yaml @@ -35,9 +35,15 @@ vault: - name: vault-nfs-storage-backup persistentVolumeClaim: claimName: vault-nfs-storage-backup + - name: vault-nfs-storage-backup + persistentVolumeClaim: + claimName: vault-nfs-storage-backup volumeMounts: - - mountPath: /opt/backups/ + - mountPath: /opt/backups-old/ name: vault-nfs-storage-backup + readOnly: true + - mountPath: /opt/backups/ + name: vault-storage-backup readOnly: false affinity: | podAntiAffinity: @@ -176,26 +182,15 @@ snapshot: - /bin/ash args: - -ec - - | - apk add --no-cache jq; - echo ">> Running Vault snapshot" - export VAULT_TOKEN=$(vault write auth/approle/login role_id=$VAULT_APPROLE_ROLE_ID secret_id=$VAULT_APPROLE_SECRET_ID -format=json | jq -r .auth.client_token); - vault operator raft snapshot save /opt/backup/vault-snapshot-latest.snap; - cp /opt/backup/vault-snapshot-latest.snap /opt/backup/vault-snapshot-$(date +"%Y%m%d-%H-%M").snap; - cp /opt/backup/vault-snapshot-latest.snap /opt/backup/vault-snapshot-s3.snap; - echo ">> Completed Vault snapshot" + - /scripts/snapshot.sh envFrom: - secretRef: name: vault-snapshot-agent-token env: - name: VAULT_ADDR value: http://vault-active.vault.svc.cluster.local:8200 - resources: - requests: - cpu: 10m - memory: 64Mi containers: - s3-backup: + s3-backup-local: image: repository: d3fk/s3cmd tag: latest@sha256:ed348a0fae5723d2e62636c175baf4dfaf732a790179ca675d1f24f863d0d68f @@ -204,43 +199,121 @@ snapshot: - /bin/sh args: - -ec - - | - echo ">> Running S3 backup for Vault snapshot" - s3cmd put --no-check-md5 --no-check-certificate -v /opt/backup/vault-snapshot-s3.snap ${BUCKET}/cl01tl/cl01tl-vault-snapshots/vault-snapshot-$(date +"%Y%m%d-%H-%M").snap; - rm -f /opt/backup/vault-snapshot-s3.snap; - echo ">> Completed S3 backup for Vault snapshot" + - /scripts/backup.sh env: - name: BUCKET valueFrom: secretKeyRef: - name: vault-s3cmd-config + name: vault-s3cmd-local-config + key: BUCKET + s3-backup-remote: + image: + repository: d3fk/s3cmd + tag: latest@sha256:ed348a0fae5723d2e62636c175baf4dfaf732a790179ca675d1f24f863d0d68f + pullPolicy: IfNotPresent + command: + - /bin/sh + args: + - -ec + - /scripts/backup.sh + env: + - name: BUCKET + valueFrom: + secretKeyRef: + name: vault-s3cmd-remote-config + key: BUCKET + s3-backup-external: + image: + repository: d3fk/s3cmd + tag: latest@sha256:ed348a0fae5723d2e62636c175baf4dfaf732a790179ca675d1f24f863d0d68f + pullPolicy: IfNotPresent + command: + - /bin/sh + args: + - -ec + - /scripts/backup.sh + env: + - name: BUCKET + valueFrom: + secretKeyRef: + name: vault-s3cmd-external-config key: BUCKET - resources: - requests: - cpu: 100m - memory: 128Mi persistence: - config: - existingClaim: vault-nfs-storage-backup + snapshot-script: + enabled: true + type: configMap + name: vault-snapshot-script + defaultMode: 0755 + advancedMounts: + snapshot: + snapshot: + - path: /scripts/snapshot.sh + subPath: snapshot.sh + backup-script: + enabled: true + type: configMap + name: vault-backup-script + defaultMode: 0755 + advancedMounts: + snapshot: + s3-backup-local: + - path: /scripts/backup.sh + subPath: backup.sh + s3-backup-remote: + - path: /scripts/backup.sh + subPath: backup.sh + s3-backup-external: + - path: /scripts/backup.sh + subPath: backup.sh + s3cmd-local-config: + enabled: true + type: secret + name: vault-s3cmd-local-config + advancedMounts: + snapshot: + s3-backup-local: + - path: /root/.s3cfg + readOnly: true + mountPropagation: None + subPath: .s3cfg + s3cmd-remote-config: + enabled: true + type: secret + name: vault-s3cmd-remote-config + advancedMounts: + snapshot: + s3-backup-remote: + - path: /root/.s3cfg + readOnly: true + mountPropagation: None + subPath: .s3cfg + s3cmd-external-config: + enabled: true + type: secret + name: vault-s3cmd-external-config + advancedMounts: + snapshot: + s3-backup-external: + - path: /root/.s3cfg + readOnly: true + mountPropagation: None + subPath: .s3cfg + backup: + existingClaim: vault-storage-backup advancedMounts: snapshot: snapshot: - path: /opt/backup readOnly: false - s3-backup: + s3-backup-local: + - path: /opt/backup + readOnly: false + s3-backup-remote: + - path: /opt/backup + readOnly: false + s3-backup-external: - path: /opt/backup readOnly: false - s3cmd-config: - enabled: true - type: secret - name: vault-s3cmd-config - advancedMounts: - snapshot: - s3-backup: - - path: /root/.s3cfg - readOnly: true - mountPropagation: None - subPath: .s3cfg unseal: global: fullnameOverride: vault-unseal