Files
infrastructure/clusters/cl01tl/manifests/tubearchivist/ConfigMap-tubearchivist-valkey-init-scripts.yaml

836 lines
23 KiB
YAML

apiVersion: v1
kind: ConfigMap
metadata:
name: tubearchivist-valkey-init-scripts
labels:
helm.sh/chart: valkey-0.9.3
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/version: "9.0.3"
app.kubernetes.io/managed-by: Helm
data:
init.sh: |-
#!/bin/sh
set -eu
# Default config paths
VALKEY_CONFIG=${VALKEY_CONFIG_PATH:-/data/conf/valkey.conf}
LOGFILE="/data/init.log"
DATA_DIR="/data/conf"
# Logging function (outputs to stderr and file)
log() {
echo "$(date) $1" | tee -a "$LOGFILE" >&2
}
# Clean old log if requested
if [ "${KEEP_OLD_LOGS:-false}" != "true" ]; then
rm -f "$LOGFILE"
fi
if [ -f "$LOGFILE" ]; then
log "Detected restart of this instance ($HOSTNAME)"
fi
log "Creating configuration in $DATA_DIR..."
mkdir -p "$DATA_DIR"
rm -f "$VALKEY_CONFIG"
# Base valkey.conf
log "Generating base valkey.conf"
{
echo "port 6379"
echo "protected-mode no"
echo "bind * -::*"
echo "dir /data"
} >>"$VALKEY_CONFIG"
# Replica mode configuration
log "Configuring replication mode"
# Use POD_INDEX from Kubernetes metadata
POD_INDEX=${POD_INDEX:-0}
IS_MASTER=false
# Check if this is pod-0 (master)
if [ "$POD_INDEX" = "0" ]; then
IS_MASTER=true
log "This pod (index $POD_INDEX) is configured as MASTER"
else
log "This pod (index $POD_INDEX) is configured as REPLICA"
fi
# Configure replica settings
if [ "$IS_MASTER" = "false" ]; then
MASTER_HOST="tubearchivist-valkey-0.tubearchivist-valkey-headless.tubearchivist.svc.cluster.local"
MASTER_PORT="6379"
log "Configuring replica to follow master at $MASTER_HOST:$MASTER_PORT"
{
echo ""
echo "# Replica Configuration"
echo "replicaof $MASTER_HOST $MASTER_PORT"
echo "replica-announce-ip tubearchivist-valkey-$POD_INDEX.tubearchivist-valkey-headless.tubearchivist.svc.cluster.local"
} >>"$VALKEY_CONFIG"
fi
# Append extra configs if present
if [ -f /usr/local/etc/valkey/valkey.conf ]; then
log "Appending /usr/local/etc/valkey/valkey.conf"
cat /usr/local/etc/valkey/valkey.conf >>"$VALKEY_CONFIG"
fi
if [ -d /extravalkeyconfigs ]; then
log "Appending files in /extravalkeyconfigs/"
cat /extravalkeyconfigs/* >>"$VALKEY_CONFIG"
fi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: tubearchivist
labels:
app.kubernetes.io/controller: main
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: tubearchivist
helm.sh/chart: tubearchivist-4.6.2
namespace: tubearchivist
spec:
revisionHistoryLimit: 3
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/controller: main
app.kubernetes.io/name: tubearchivist
app.kubernetes.io/instance: tubearchivist
template:
metadata:
labels:
app.kubernetes.io/controller: main
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/name: tubearchivist
spec:
enableServiceLinks: false
serviceAccountName: default
automountServiceAccountToken: true
hostIPC: false
hostNetwork: false
hostPID: false
dnsPolicy: ClusterFirst
containers:
- image: brainicism/bgutil-ytdlp-pot-provider:1.3.1
imagePullPolicy: IfNotPresent
name: bgutil
- env:
- name: VPN_SERVICE_PROVIDER
value: protonvpn
- name: VPN_TYPE
value: wireguard
- name: WIREGUARD_PRIVATE_KEY
valueFrom:
secretKeyRef:
key: private-key
name: tubearchivist-wireguard-conf
- name: UPDATER_PROTONVPN_EMAIL
valueFrom:
secretKeyRef:
key: proton-email
name: tubearchivist-wireguard-conf
- name: UPDATER_PROTONVPN_PASSWORD
valueFrom:
secretKeyRef:
key: proton-password
name: tubearchivist-wireguard-conf
- name: FIREWALL_OUTBOUND_SUBNETS
value: 10.0.0.0/8
- name: FIREWALL_INPUT_PORTS
value: 80,8000,24000
- name: DNS_UPSTREAM_RESOLVER_TYPE
value: dot
- name: HTTPPROXY
value: "off"
- name: SHADOWSOCKS
value: "off"
image: ghcr.io/qdm12/gluetun:v3.41.1@sha256:1a5bf4b4820a879cdf8d93d7ef0d2d963af56670c9ebff8981860b6804ebc8ab
imagePullPolicy: IfNotPresent
lifecycle:
postStart:
exec:
command:
- /bin/sh
- -c
- (ip rule del table 51820; ip -6 rule del table 51820) || true
livenessProbe:
exec:
command:
- /gluetun-entrypoint
- healthcheck
failureThreshold: 5
initialDelaySeconds: 30
periodSeconds: 30
successThreshold: 1
timeoutSeconds: 15
name: gluetun
resources:
limits:
devic.es/tun: "1"
requests:
cpu: 10m
devic.es/tun: "1"
memory: 128Mi
securityContext:
capabilities:
add:
- NET_ADMIN
- SYS_MODULE
privileged: true
- env:
- name: TZ
value: America/Chicago
- name: HOST_UID
value: "1000"
- name: HOST_GID
value: "1000"
- name: ES_URL
value: https://elasticsearch-tubearchivist-es-http.tubearchivist:9200
- name: ES_DISABLE_VERIFY_SSL
value: "true"
- name: REDIS_CON
value: redis://tubearchivist-valkey.tubearchivist:6379
- name: TA_HOST
value: https://tubearchivist.alexlebens.net http://tubearchivist.tubearchivist:80/
- name: TA_PORT
value: "24000"
- name: TA_USERNAME
value: admin
envFrom:
- secretRef:
name: tubearchivist-config-secret
image: bbilly1/tubearchivist:v0.5.9
imagePullPolicy: IfNotPresent
name: main
resources:
requests:
cpu: 10m
memory: 1Gi
volumeMounts:
- mountPath: /cache
name: data
- mountPath: /youtube
name: youtube
volumes:
- name: data
persistentVolumeClaim:
claimName: tubearchivist
- name: youtube
persistentVolumeClaim:
claimName: tubearchivist-nfs-storage
---
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
name: elasticsearch-tubearchivist
namespace: tubearchivist
labels:
app.kubernetes.io/name: elasticsearch-tubearchivist
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/part-of: tubearchivist
spec:
version: 8.19.8
auth:
fileRealm:
- secretName: tubearchivist-elasticsearch-secret
nodeSets:
- name: default
count: 2
config:
node.store.allow_mmap: false
path.repo: /usr/share/elasticsearch/data/snapshot
podTemplate:
spec:
volumes:
- name: tubearchivist-snapshot-nfs-storage
nfs:
path: /volume2/Storage/TubeArchivist
server: synologybond.alexlebens.net
containers:
- name: elasticsearch
volumeMounts:
- name: tubearchivist-snapshot-nfs-storage
mountPath: /usr/share/elasticsearch/data/snapshot
volumeClaimTemplates:
- metadata:
name: elasticsearch-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
storageClassName: ceph-block
---
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: tubearchivist-config-secret
namespace: tubearchivist
labels:
app.kubernetes.io/name: tubearchivist-config-secret
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/part-of: tubearchivist
spec:
secretStoreRef:
kind: ClusterSecretStore
name: vault
data:
- secretKey: ELASTIC_PASSWORD
remoteRef:
conversionStrategy: Default
decodingStrategy: None
key: /cl01tl/tubearchivist/env
metadataPolicy: None
property: ELASTIC_PASSWORD
- secretKey: TA_PASSWORD
remoteRef:
conversionStrategy: Default
decodingStrategy: None
key: /cl01tl/tubearchivist/env
metadataPolicy: None
property: TA_PASSWORD
---
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: tubearchivist-elasticsearch-secret
namespace: tubearchivist
labels:
app.kubernetes.io/name: tubearchivist-elasticsearch-secret
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/part-of: tubearchivist
spec:
secretStoreRef:
kind: ClusterSecretStore
name: vault
data:
- secretKey: username
remoteRef:
conversionStrategy: Default
decodingStrategy: None
key: /cl01tl/tubearchivist/elasticsearch
metadataPolicy: None
property: username
- secretKey: password
remoteRef:
conversionStrategy: Default
decodingStrategy: None
key: /cl01tl/tubearchivist/elasticsearch
metadataPolicy: None
property: password
- secretKey: roles
remoteRef:
conversionStrategy: Default
decodingStrategy: None
key: /cl01tl/tubearchivist/elasticsearch
metadataPolicy: None
property: roles
---
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: tubearchivist-wireguard-conf
namespace: tubearchivist
labels:
app.kubernetes.io/name: tubearchivist-wireguard-conf
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/part-of: tubearchivist
spec:
secretStoreRef:
kind: ClusterSecretStore
name: vault
data:
- secretKey: private-key
remoteRef:
conversionStrategy: Default
decodingStrategy: None
key: /protonvpn/conf/cl01tl
metadataPolicy: None
property: private-key
- secretKey: proton-email
remoteRef:
conversionStrategy: Default
decodingStrategy: None
key: /protonvpn/conf/cl01tl
metadataPolicy: None
property: email
- secretKey: proton-password
remoteRef:
conversionStrategy: Default
decodingStrategy: None
key: /protonvpn/conf/cl01tl
metadataPolicy: None
property: password
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: tubearchivist
labels:
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: tubearchivist
helm.sh/chart: tubearchivist-4.6.2
namespace: tubearchivist
spec:
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: traefik-gateway
namespace: traefik
hostnames:
- "tubearchivist.alexlebens.net"
rules:
- backendRefs:
- group: ""
kind: Service
name: tubearchivist
namespace: tubearchivist
port: 80
weight: 100
matches:
- path:
type: PathPrefix
value: /
---
apiVersion: v1
kind: Namespace
metadata:
name: tubearchivist
labels:
app.kubernetes.io/name: tubearchivist
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/part-of: tubearchivist
pod-security.kubernetes.io/audit: privileged
pod-security.kubernetes.io/enforce: privileged
pod-security.kubernetes.io/warn: privileged
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: tubearchivist-nfs-storage
namespace: tubearchivist
labels:
app.kubernetes.io/name: tubearchivist-nfs-storage
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/part-of: tubearchivist
spec:
persistentVolumeReclaimPolicy: Retain
storageClassName: nfs-client
capacity:
storage: 1Gi
accessModes:
- ReadWriteMany
nfs:
path: /volume2/Storage/YouTube
server: synologybond.alexlebens.net
mountOptions:
- vers=4
- minorversion=1
- noac
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: tubearchivist-nfs-storage
namespace: tubearchivist
labels:
app.kubernetes.io/name: tubearchivist-nfs-storage
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/part-of: tubearchivist
spec:
volumeName: tubearchivist-nfs-storage
storageClassName: nfs-client
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Gi
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: tubearchivist
labels:
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: tubearchivist
helm.sh/chart: tubearchivist-4.6.2
annotations:
helm.sh/resource-policy: keep
namespace: tubearchivist
spec:
accessModes:
- "ReadWriteOnce"
resources:
requests:
storage: "40Gi"
storageClassName: "ceph-block"
---
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
name: tubearchivist-valkey
labels:
helm.sh/chart: valkey-0.9.3
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/version: "9.0.3"
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/part-of: valkey
app.kubernetes.io/component: podmonitor
spec:
podMetricsEndpoints:
- port: metrics
interval: 30s
namespaceSelector:
matchNames:
- tubearchivist
selector:
matchLabels:
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: tubearchivist-valkey
labels:
helm.sh/chart: valkey-0.9.3
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/version: "9.0.3"
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/part-of: valkey
spec:
groups:
- name: tubearchivist-valkey
rules:
- alert: ValkeyDown
annotations:
description: Valkey instance {{ $labels.instance }} is down.
summary: Valkey instance {{ $labels.instance }} down
expr: |
redis_up{service="tubearchivist-valkey-metrics"} == 0
for: 2m
labels:
severity: error
- alert: ValkeyMemoryHigh
annotations:
description: |
Valkey instance {{ $labels.instance }} is using {{ $value }}% of its available memory.
summary: Valkey instance {{ $labels.instance }} is using too much memory
expr: |
redis_memory_used_bytes{service="tubearchivist-valkey-metrics"} * 100
/
redis_memory_max_bytes{service="tubearchivist-valkey-metrics"}
> 90 <= 100
for: 2m
labels:
severity: error
- alert: ValkeyKeyEviction
annotations:
description: |
Valkey instance {{ $labels.instance }} has evicted {{ $value }} keys in the last 5 minutes.
summary: Valkey instance {{ $labels.instance }} has evicted keys
expr: |
increase(redis_evicted_keys_total{service="tubearchivist-valkey-metrics"}[5m]) > 0
for: 1s
labels:
severity: error
---
apiVersion: v1
kind: Service
metadata:
name: tubearchivist-valkey-headless
labels:
helm.sh/chart: valkey-0.9.3
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/version: "9.0.3"
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: headless
spec:
type: ClusterIP
clusterIP: None
publishNotReadyAddresses: true
ports:
- name: tcp
port: 6379
targetPort: tcp
protocol: TCP
selector:
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
---
apiVersion: v1
kind: Service
metadata:
name: tubearchivist-valkey-metrics
labels:
helm.sh/chart: valkey-0.9.3
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/version: "9.0.3"
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: metrics
app.kubernetes.io/part-of: valkey
annotations:
spec:
type: ClusterIP
ports:
- name: metrics
port: 9121
protocol: TCP
targetPort: metrics
selector:
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
---
apiVersion: v1
kind: Service
metadata:
name: tubearchivist-valkey-read
labels:
helm.sh/chart: valkey-0.9.3
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/version: "9.0.3"
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: read
spec:
type: ClusterIP
ports:
- name: tcp
port: 6379
targetPort: tcp
protocol: TCP
selector:
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
---
apiVersion: v1
kind: Service
metadata:
name: tubearchivist-valkey
labels:
helm.sh/chart: valkey-0.9.3
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/version: "9.0.3"
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: primary
spec:
type: ClusterIP
ports:
- port: 6379
targetPort: tcp
protocol: TCP
name: tcp
selector:
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
statefulset.kubernetes.io/pod-name: tubearchivist-valkey-0
---
apiVersion: v1
kind: Service
metadata:
name: tubearchivist
labels:
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: tubearchivist
app.kubernetes.io/service: tubearchivist
helm.sh/chart: tubearchivist-4.6.2
namespace: tubearchivist
spec:
type: ClusterIP
ports:
- port: 80
targetPort: 24000
protocol: TCP
name: http
selector:
app.kubernetes.io/controller: main
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/name: tubearchivist
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: tubearchivist-valkey
labels:
helm.sh/chart: valkey-0.9.3
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/version: "9.0.3"
app.kubernetes.io/managed-by: Helm
automountServiceAccountToken: false
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: tubearchivist-valkey
labels:
helm.sh/chart: valkey-0.9.3
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/version: "9.0.3"
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/part-of: valkey
app.kubernetes.io/component: service-monitor
spec:
endpoints:
- port: metrics
interval: 30s
namespaceSelector:
matchNames:
- tubearchivist
selector:
matchLabels:
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/component: metrics
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: tubearchivist-valkey
labels:
helm.sh/chart: valkey-0.9.3
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
app.kubernetes.io/version: "9.0.3"
app.kubernetes.io/managed-by: Helm
spec:
serviceName: tubearchivist-valkey-headless
replicas: 3
podManagementPolicy: OrderedReady
selector:
matchLabels:
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
volumeClaimTemplates:
- metadata:
name: valkey-data
spec:
accessModes:
- ReadWriteOnce
storageClassName: "ceph-block"
resources:
requests:
storage: "10Gi"
template:
metadata:
labels:
app.kubernetes.io/name: valkey
app.kubernetes.io/instance: tubearchivist
annotations:
checksum/initconfig: "98c1a271c9c183213a1aa113039e1a4e"
spec:
automountServiceAccountToken: false
serviceAccountName: tubearchivist-valkey
securityContext:
fsGroup: 1000
runAsGroup: 1000
runAsUser: 1000
initContainers:
- name: tubearchivist-valkey-init
image: docker.io/valkey/valkey:9.0.3
imagePullPolicy: IfNotPresent
securityContext:
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
command: ["/scripts/init.sh"]
env:
- name: POD_INDEX
valueFrom:
fieldRef:
fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
volumeMounts:
- name: valkey-data
mountPath: /data
- name: scripts
mountPath: /scripts
containers:
- name: tubearchivist-valkey
image: docker.io/valkey/valkey:9.0.3
imagePullPolicy: IfNotPresent
command: ["valkey-server"]
args: ["/data/conf/valkey.conf"]
securityContext:
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
env:
- name: POD_INDEX
valueFrom:
fieldRef:
fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
- name: VALKEY_LOGLEVEL
value: "notice"
ports:
- name: tcp
containerPort: 6379
protocol: TCP
startupProbe:
exec:
command: ["sh", "-c", "valkey-cli ping"]
livenessProbe:
exec:
command: ["sh", "-c", "valkey-cli ping"]
resources:
requests:
cpu: 100m
memory: 1Gi
volumeMounts:
- name: valkey-data
mountPath: /data
- name: metrics
image: ghcr.io/oliver006/redis_exporter:v1.82.0
imagePullPolicy: "IfNotPresent"
ports:
- name: metrics
containerPort: 9121
startupProbe:
tcpSocket:
port: metrics
livenessProbe:
tcpSocket:
port: metrics
readinessProbe:
httpGet:
path: /
port: metrics
resources:
requests:
cpu: 10m
memory: 64M
env:
- name: REDIS_ALIAS
value: tubearchivist-valkey
volumes:
- name: scripts
configMap:
name: tubearchivist-valkey-init-scripts
defaultMode: 0555