alexlebens/infrastructure
1
0
Fork 0
Code Issues 1 Pull Requests 2 Actions Activity

chore: Update manifests after change

Browse Source
This commit is contained in:
gitea-bot
2025-12-04 04:54:08 +00:00
parent dad1b4623c
commit bfe0e18539
2066 changed files with 378996 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

743
clusters/cl01tl/manifests/argocd/ConfigMap-argocd-redis-ha-configmap.yml Normal file
View File

@@ -0,0 +1,743 @@
---
# Source: argocd/charts/argo-cd/charts/redis-ha/templates/redis-ha-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: argocd-redis-ha-configmap
namespace: "argocd"
labels:
heritage: Helm
release: argocd
chart: redis-ha-4.34.11
app: argocd-redis-ha
data:
redis.conf: |
dir "/data"
port 6379
rename-command FLUSHDB ""
rename-command FLUSHALL ""
maxmemory 0
maxmemory-policy volatile-lru
min-replicas-max-lag 5
min-replicas-to-write 1
rdbchecksum yes
rdbcompression yes
repl-diskless-sync yes
save ""
sentinel.conf: |
dir "/data"
port 26379
sentinel down-after-milliseconds argocd 10000
sentinel failover-timeout argocd 180000
maxclients 10000
sentinel parallel-syncs argocd 5
init.sh: |
echo "$(date) Start..."
HOSTNAME="$(hostname)"
INDEX="${HOSTNAME##*-}"
SENTINEL_PORT=26379
ANNOUNCE_IP=''
MASTER=''
MASTER_GROUP="argocd"
QUORUM="2"
REDIS_CONF=/data/conf/redis.conf
REDIS_PORT=6379
REDIS_TLS_PORT=
SENTINEL_CONF=/data/conf/sentinel.conf
SENTINEL_TLS_PORT=
SERVICE=argocd-redis-ha
SENTINEL_TLS_REPLICATION_ENABLED=false
REDIS_TLS_REPLICATION_ENABLED=false
set -eu
sentinel_get_master() {
set +e
if [ "$SENTINEL_PORT" -eq 0 ]; then
redis-cli -h "${SERVICE}" -p "${SENTINEL_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key sentinel get-master-addr-by-name "${MASTER_GROUP}" |\
grep -E '((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?s*$))'
else
redis-cli -h "${SERVICE}" -p "${SENTINEL_PORT}" sentinel get-master-addr-by-name "${MASTER_GROUP}" |\
grep -E '((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?s*$))'
fi
set -e
}
sentinel_get_master_retry() {
master=''
retry=${1}
sleep=3
for i in $(seq 1 "${retry}"); do
master=$(sentinel_get_master)
if [ -n "${master}" ]; then
break
fi
sleep $((sleep + i))
done
echo "${master}"
}
identify_master() {
echo "Identifying redis master (get-master-addr-by-name).."
echo " using sentinel (argocd-redis-ha), sentinel group name (argocd)"
MASTER="$(sentinel_get_master_retry 3)"
if [ -n "${MASTER}" ]; then
echo " $(date) Found redis master (${MASTER})"
else
echo " $(date) Did not find redis master (${MASTER})"
fi
}
sentinel_update() {
echo "Updating sentinel config.."
echo " evaluating sentinel id (\${SENTINEL_ID_${INDEX}})"
eval MY_SENTINEL_ID="\$SENTINEL_ID_${INDEX}"
echo " sentinel id (${MY_SENTINEL_ID}), sentinel grp (${MASTER_GROUP}), quorum (${QUORUM})"
sed -i "1s/^/sentinel myid ${MY_SENTINEL_ID}\\n/" "${SENTINEL_CONF}"
if [ "$SENTINEL_TLS_REPLICATION_ENABLED" = true ]; then
echo " redis master (${1}:${REDIS_TLS_PORT})"
sed -i "2s/^/sentinel monitor ${MASTER_GROUP} ${1} ${REDIS_TLS_PORT} ${QUORUM} \\n/" "${SENTINEL_CONF}"
else
echo " redis master (${1}:${REDIS_PORT})"
sed -i "2s/^/sentinel monitor ${MASTER_GROUP} ${1} ${REDIS_PORT} ${QUORUM} \\n/" "${SENTINEL_CONF}"
fi
echo "sentinel announce-ip ${ANNOUNCE_IP}" >> ${SENTINEL_CONF}
if [ "$SENTINEL_PORT" -eq 0 ]; then
echo " announce (${ANNOUNCE_IP}:${SENTINEL_TLS_PORT})"
echo "sentinel announce-port ${SENTINEL_TLS_PORT}" >> ${SENTINEL_CONF}
else
echo " announce (${ANNOUNCE_IP}:${SENTINEL_PORT})"
echo "sentinel announce-port ${SENTINEL_PORT}" >> ${SENTINEL_CONF}
fi
}
redis_update() {
echo "Updating redis config.."
if [ "$REDIS_TLS_REPLICATION_ENABLED" = true ]; then
echo " we are slave of redis master (${1}:${REDIS_TLS_PORT})"
echo "slaveof ${1} ${REDIS_TLS_PORT}" >> "${REDIS_CONF}"
echo "slave-announce-port ${REDIS_TLS_PORT}" >> ${REDIS_CONF}
else
echo " we are slave of redis master (${1}:${REDIS_PORT})"
echo "slaveof ${1} ${REDIS_PORT}" >> "${REDIS_CONF}"
echo "slave-announce-port ${REDIS_PORT}" >> ${REDIS_CONF}
fi
echo "slave-announce-ip ${ANNOUNCE_IP}" >> ${REDIS_CONF}
}
copy_config() {
echo "Copying default redis config.."
echo " to '${REDIS_CONF}'"
cp /readonly-config/redis.conf "${REDIS_CONF}"
echo "Copying default sentinel config.."
echo " to '${SENTINEL_CONF}'"
cp /readonly-config/sentinel.conf "${SENTINEL_CONF}"
}
setup_defaults() {
echo "Setting up defaults.."
echo " using statefulset index (${INDEX})"
if [ "${INDEX}" = "0" ]; then
echo "Setting this pod as master for redis and sentinel.."
echo " using announce (${ANNOUNCE_IP})"
redis_update "${ANNOUNCE_IP}"
sentinel_update "${ANNOUNCE_IP}"
echo " make sure ${ANNOUNCE_IP} is not a slave (slaveof no one)"
sed -i "s/^.*slaveof.*//" "${REDIS_CONF}"
else
echo "Getting redis master ip.."
echo " blindly assuming (${SERVICE}-announce-0) or (${SERVICE}-server-0) are master"
DEFAULT_MASTER="$(getent_hosts 0 | awk '{ print $1 }')"
if [ -z "${DEFAULT_MASTER}" ]; then
echo "Error: Unable to resolve redis master (getent hosts)."
exit 1
fi
echo " identified redis (may be redis master) ip (${DEFAULT_MASTER})"
echo "Setting default slave config for redis and sentinel.."
echo " using master ip (${DEFAULT_MASTER})"
redis_update "${DEFAULT_MASTER}"
sentinel_update "${DEFAULT_MASTER}"
fi
}
redis_ping() {
set +e
if [ "$REDIS_PORT" -eq 0 ]; then
redis-cli -h "${MASTER}" -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key ping
else
redis-cli -h "${MASTER}" -p "${REDIS_PORT}" ping
fi
set -e
}
redis_ping_retry() {
ping=''
retry=${1}
sleep=3
for i in $(seq 1 "${retry}"); do
if [ "$(redis_ping)" = "PONG" ]; then
ping='PONG'
break
fi
sleep $((sleep + i))
MASTER=$(sentinel_get_master)
done
echo "${ping}"
}
find_master() {
echo "Verifying redis master.."
if [ "$REDIS_PORT" -eq 0 ]; then
echo " ping (${MASTER}:${REDIS_TLS_PORT})"
else
echo " ping (${MASTER}:${REDIS_PORT})"
fi
if [ "$(redis_ping_retry 3)" != "PONG" ]; then
echo " $(date) Can't ping redis master (${MASTER})"
echo "Attempting to force failover (sentinel failover).."
if [ "$SENTINEL_PORT" -eq 0 ]; then
echo " on sentinel (${SERVICE}:${SENTINEL_TLS_PORT}), sentinel grp (${MASTER_GROUP})"
if redis-cli -h "${SERVICE}" -p "${SENTINEL_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key sentinel failover "${MASTER_GROUP}" | grep -q 'NOGOODSLAVE' ; then
echo " $(date) Failover returned with 'NOGOODSLAVE'"
echo "Setting defaults for this pod.."
setup_defaults
return 0
fi
else
echo " on sentinel (${SERVICE}:${SENTINEL_PORT}), sentinel grp (${MASTER_GROUP})"
if redis-cli -h "${SERVICE}" -p "${SENTINEL_PORT}" sentinel failover "${MASTER_GROUP}" | grep -q 'NOGOODSLAVE' ; then
echo " $(date) Failover returned with 'NOGOODSLAVE'"
echo "Setting defaults for this pod.."
setup_defaults
return 0
fi
fi
echo "Hold on for 10sec"
sleep 10
echo "We should get redis master's ip now. Asking (get-master-addr-by-name).."
if [ "$SENTINEL_PORT" -eq 0 ]; then
echo " sentinel (${SERVICE}:${SENTINEL_TLS_PORT}), sentinel grp (${MASTER_GROUP})"
else
echo " sentinel (${SERVICE}:${SENTINEL_PORT}), sentinel grp (${MASTER_GROUP})"
fi
MASTER="$(sentinel_get_master)"
if [ "${MASTER}" ]; then
echo " $(date) Found redis master (${MASTER})"
echo "Updating redis and sentinel config.."
sentinel_update "${MASTER}"
redis_update "${MASTER}"
else
echo "$(date) Error: Could not failover, exiting..."
exit 1
fi
else
echo " $(date) Found reachable redis master (${MASTER})"
echo "Updating redis and sentinel config.."
sentinel_update "${MASTER}"
redis_update "${MASTER}"
fi
}
redis_ro_update() {
echo "Updating read-only redis config.."
echo " redis.conf set 'replica-priority 0'"
echo "replica-priority 0" >> ${REDIS_CONF}
}
getent_hosts() {
index=${1:-${INDEX}}
service="${SERVICE}-announce-${index}"
host=$(getent hosts "${service}")
echo "${host}"
}
identify_announce_ip() {
echo "Identify announce ip for this pod.."
echo " using (${SERVICE}-announce-${INDEX}) or (${SERVICE}-server-${INDEX})"
ANNOUNCE_IP=$(getent_hosts | awk '{ print $1 }')
echo " identified announce (${ANNOUNCE_IP})"
}
mkdir -p /data/conf/
echo "Initializing config.."
copy_config
# where is redis master
identify_master
identify_announce_ip
if [ -z "${ANNOUNCE_IP}" ]; then
"Error: Could not resolve the announce ip for this pod"
exit 1
elif [ "${MASTER}" ]; then
find_master
else
setup_defaults
fi
if [ "${AUTH:-}" ]; then
echo "Setting redis auth values.."
ESCAPED_AUTH=$(echo "${AUTH}" | sed -e 's/[\/&]/\\&/g');
sed -i "s/replace-default-auth/${ESCAPED_AUTH}/" "${REDIS_CONF}" "${SENTINEL_CONF}"
fi
if [ "${SENTINELAUTH:-}" ]; then
echo "Setting sentinel auth values"
ESCAPED_AUTH_SENTINEL=$(echo "$SENTINELAUTH" | sed -e 's/[\/&]/\\&/g');
sed -i "s/replace-default-sentinel-auth/${ESCAPED_AUTH_SENTINEL}/" "$SENTINEL_CONF"
fi
echo "$(date) Ready..."
fix-split-brain.sh: |
HOSTNAME="$(hostname)"
INDEX="${HOSTNAME##*-}"
SENTINEL_PORT=26379
ANNOUNCE_IP=''
MASTER=''
MASTER_GROUP="argocd"
QUORUM="2"
REDIS_CONF=/data/conf/redis.conf
REDIS_PORT=6379
REDIS_TLS_PORT=
SENTINEL_CONF=/data/conf/sentinel.conf
SENTINEL_TLS_PORT=
SERVICE=argocd-redis-ha
SENTINEL_TLS_REPLICATION_ENABLED=false
REDIS_TLS_REPLICATION_ENABLED=false
ROLE=''
REDIS_MASTER=''
set -eu
sentinel_get_master() {
set +e
if [ "$SENTINEL_PORT" -eq 0 ]; then
redis-cli -h "${SERVICE}" -p "${SENTINEL_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key sentinel get-master-addr-by-name "${MASTER_GROUP}" |\
grep -E '((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?s*$))'
else
redis-cli -h "${SERVICE}" -p "${SENTINEL_PORT}" sentinel get-master-addr-by-name "${MASTER_GROUP}" |\
grep -E '((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?s*$))'
fi
set -e
}
sentinel_get_master_retry() {
master=''
retry=${1}
sleep=3
for i in $(seq 1 "${retry}"); do
master=$(sentinel_get_master)
if [ -n "${master}" ]; then
break
fi
sleep $((sleep + i))
done
echo "${master}"
}
identify_master() {
echo "Identifying redis master (get-master-addr-by-name).."
echo " using sentinel (argocd-redis-ha), sentinel group name (argocd)"
MASTER="$(sentinel_get_master_retry 3)"
if [ -n "${MASTER}" ]; then
echo " $(date) Found redis master (${MASTER})"
else
echo " $(date) Did not find redis master (${MASTER})"
fi
}
sentinel_update() {
echo "Updating sentinel config.."
echo " evaluating sentinel id (\${SENTINEL_ID_${INDEX}})"
eval MY_SENTINEL_ID="\$SENTINEL_ID_${INDEX}"
echo " sentinel id (${MY_SENTINEL_ID}), sentinel grp (${MASTER_GROUP}), quorum (${QUORUM})"
sed -i "1s/^/sentinel myid ${MY_SENTINEL_ID}\\n/" "${SENTINEL_CONF}"
if [ "$SENTINEL_TLS_REPLICATION_ENABLED" = true ]; then
echo " redis master (${1}:${REDIS_TLS_PORT})"
sed -i "2s/^/sentinel monitor ${MASTER_GROUP} ${1} ${REDIS_TLS_PORT} ${QUORUM} \\n/" "${SENTINEL_CONF}"
else
echo " redis master (${1}:${REDIS_PORT})"
sed -i "2s/^/sentinel monitor ${MASTER_GROUP} ${1} ${REDIS_PORT} ${QUORUM} \\n/" "${SENTINEL_CONF}"
fi
echo "sentinel announce-ip ${ANNOUNCE_IP}" >> ${SENTINEL_CONF}
if [ "$SENTINEL_PORT" -eq 0 ]; then
echo " announce (${ANNOUNCE_IP}:${SENTINEL_TLS_PORT})"
echo "sentinel announce-port ${SENTINEL_TLS_PORT}" >> ${SENTINEL_CONF}
else
echo " announce (${ANNOUNCE_IP}:${SENTINEL_PORT})"
echo "sentinel announce-port ${SENTINEL_PORT}" >> ${SENTINEL_CONF}
fi
}
redis_update() {
echo "Updating redis config.."
if [ "$REDIS_TLS_REPLICATION_ENABLED" = true ]; then
echo " we are slave of redis master (${1}:${REDIS_TLS_PORT})"
echo "slaveof ${1} ${REDIS_TLS_PORT}" >> "${REDIS_CONF}"
echo "slave-announce-port ${REDIS_TLS_PORT}" >> ${REDIS_CONF}
else
echo " we are slave of redis master (${1}:${REDIS_PORT})"
echo "slaveof ${1} ${REDIS_PORT}" >> "${REDIS_CONF}"
echo "slave-announce-port ${REDIS_PORT}" >> ${REDIS_CONF}
fi
echo "slave-announce-ip ${ANNOUNCE_IP}" >> ${REDIS_CONF}
}
copy_config() {
echo "Copying default redis config.."
echo " to '${REDIS_CONF}'"
cp /readonly-config/redis.conf "${REDIS_CONF}"
echo "Copying default sentinel config.."
echo " to '${SENTINEL_CONF}'"
cp /readonly-config/sentinel.conf "${SENTINEL_CONF}"
}
setup_defaults() {
echo "Setting up defaults.."
echo " using statefulset index (${INDEX})"
if [ "${INDEX}" = "0" ]; then
echo "Setting this pod as master for redis and sentinel.."
echo " using announce (${ANNOUNCE_IP})"
redis_update "${ANNOUNCE_IP}"
sentinel_update "${ANNOUNCE_IP}"
echo " make sure ${ANNOUNCE_IP} is not a slave (slaveof no one)"
sed -i "s/^.*slaveof.*//" "${REDIS_CONF}"
else
echo "Getting redis master ip.."
echo " blindly assuming (${SERVICE}-announce-0) or (${SERVICE}-server-0) are master"
DEFAULT_MASTER="$(getent_hosts 0 | awk '{ print $1 }')"
if [ -z "${DEFAULT_MASTER}" ]; then
echo "Error: Unable to resolve redis master (getent hosts)."
exit 1
fi
echo " identified redis (may be redis master) ip (${DEFAULT_MASTER})"
echo "Setting default slave config for redis and sentinel.."
echo " using master ip (${DEFAULT_MASTER})"
redis_update "${DEFAULT_MASTER}"
sentinel_update "${DEFAULT_MASTER}"
fi
}
redis_ping() {
set +e
if [ "$REDIS_PORT" -eq 0 ]; then
redis-cli -h "${MASTER}" -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key ping
else
redis-cli -h "${MASTER}" -p "${REDIS_PORT}" ping
fi
set -e
}
redis_ping_retry() {
ping=''
retry=${1}
sleep=3
for i in $(seq 1 "${retry}"); do
if [ "$(redis_ping)" = "PONG" ]; then
ping='PONG'
break
fi
sleep $((sleep + i))
MASTER=$(sentinel_get_master)
done
echo "${ping}"
}
find_master() {
echo "Verifying redis master.."
if [ "$REDIS_PORT" -eq 0 ]; then
echo " ping (${MASTER}:${REDIS_TLS_PORT})"
else
echo " ping (${MASTER}:${REDIS_PORT})"
fi
if [ "$(redis_ping_retry 3)" != "PONG" ]; then
echo " $(date) Can't ping redis master (${MASTER})"
echo "Attempting to force failover (sentinel failover).."
if [ "$SENTINEL_PORT" -eq 0 ]; then
echo " on sentinel (${SERVICE}:${SENTINEL_TLS_PORT}), sentinel grp (${MASTER_GROUP})"
if redis-cli -h "${SERVICE}" -p "${SENTINEL_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key sentinel failover "${MASTER_GROUP}" | grep -q 'NOGOODSLAVE' ; then
echo " $(date) Failover returned with 'NOGOODSLAVE'"
echo "Setting defaults for this pod.."
setup_defaults
return 0
fi
else
echo " on sentinel (${SERVICE}:${SENTINEL_PORT}), sentinel grp (${MASTER_GROUP})"
if redis-cli -h "${SERVICE}" -p "${SENTINEL_PORT}" sentinel failover "${MASTER_GROUP}" | grep -q 'NOGOODSLAVE' ; then
echo " $(date) Failover returned with 'NOGOODSLAVE'"
echo "Setting defaults for this pod.."
setup_defaults
return 0
fi
fi
echo "Hold on for 10sec"
sleep 10
echo "We should get redis master's ip now. Asking (get-master-addr-by-name).."
if [ "$SENTINEL_PORT" -eq 0 ]; then
echo " sentinel (${SERVICE}:${SENTINEL_TLS_PORT}), sentinel grp (${MASTER_GROUP})"
else
echo " sentinel (${SERVICE}:${SENTINEL_PORT}), sentinel grp (${MASTER_GROUP})"
fi
MASTER="$(sentinel_get_master)"
if [ "${MASTER}" ]; then
echo " $(date) Found redis master (${MASTER})"
echo "Updating redis and sentinel config.."
sentinel_update "${MASTER}"
redis_update "${MASTER}"
else
echo "$(date) Error: Could not failover, exiting..."
exit 1
fi
else
echo " $(date) Found reachable redis master (${MASTER})"
echo "Updating redis and sentinel config.."
sentinel_update "${MASTER}"
redis_update "${MASTER}"
fi
}
redis_ro_update() {
echo "Updating read-only redis config.."
echo " redis.conf set 'replica-priority 0'"
echo "replica-priority 0" >> ${REDIS_CONF}
}
getent_hosts() {
index=${1:-${INDEX}}
service="${SERVICE}-announce-${index}"
host=$(getent hosts "${service}")
echo "${host}"
}
identify_announce_ip() {
echo "Identify announce ip for this pod.."
echo " using (${SERVICE}-announce-${INDEX}) or (${SERVICE}-server-${INDEX})"
ANNOUNCE_IP=$(getent_hosts | awk '{ print $1 }')
echo " identified announce (${ANNOUNCE_IP})"
}
redis_role() {
set +e
if [ "$REDIS_PORT" -eq 0 ]; then
ROLE=$(redis-cli -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key info | grep role | sed 's/role://' | sed 's/\r//')
else
ROLE=$(redis-cli -p "${REDIS_PORT}" info | grep role | sed 's/role://' | sed 's/\r//')
fi
set -e
}
identify_redis_master() {
set +e
if [ "$REDIS_PORT" -eq 0 ]; then
REDIS_MASTER=$(redis-cli -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key info | grep master_host | sed 's/master_host://' | sed 's/\r//')
else
REDIS_MASTER=$(redis-cli -p "${REDIS_PORT}" info | grep master_host | sed 's/master_host://' | sed 's/\r//')
fi
set -e
}
reinit() {
set +e
sh /readonly-config/init.sh
if [ "$REDIS_PORT" -eq 0 ]; then
echo "shutdown" | redis-cli -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key
else
echo "shutdown" | redis-cli -p "${REDIS_PORT}"
fi
set -e
}
identify_announce_ip
while [ -z "${ANNOUNCE_IP}" ]; do
echo "Error: Could not resolve the announce ip for this pod."
sleep 30
identify_announce_ip
done
trap "exit 0" TERM
while true; do
sleep 60
# where is redis master
identify_master
if [ "$MASTER" = "$ANNOUNCE_IP" ]; then
redis_role
if [ "$ROLE" != "master" ]; then
echo "waiting for redis to become master"
sleep 10
identify_master
redis_role
echo "Redis role is $ROLE, expected role is master. No need to reinitialize."
if [ "$ROLE" != "master" ]; then
echo "Redis role is $ROLE, expected role is master, reinitializing"
reinit
fi
fi
elif [ "${MASTER}" ]; then
identify_redis_master
if [ "$REDIS_MASTER" != "$MASTER" ]; then
echo "Redis master and local master are not the same. waiting."
sleep 10
identify_master
identify_redis_master
echo "Redis master is ${MASTER}, expected master is ${REDIS_MASTER}. No need to reinitialize."
if [ "${REDIS_MASTER}" != "${MASTER}" ]; then
echo "Redis master is ${MASTER}, expected master is ${REDIS_MASTER}, reinitializing"
reinit
fi
fi
fi
done
haproxy.cfg: |
defaults REDIS
mode tcp
timeout connect 4s
timeout server 330s
timeout client 330s
timeout check 2s
listen health_check_http_url
bind [::]:8888 v4v6
mode http
monitor-uri /healthz
option dontlognull
# Check Sentinel and whether they are nominated master
backend check_if_redis_is_master_0
mode tcp
option tcp-check
tcp-check connect
tcp-check send PING\r\n
tcp-check expect string +PONG
tcp-check send SENTINEL\ get-master-addr-by-name\ argocd\r\n
tcp-check expect string REPLACE_ANNOUNCE0
tcp-check send QUIT\r\n
server R0 argocd-redis-ha-announce-0:26379 check inter 1s
server R1 argocd-redis-ha-announce-1:26379 check inter 1s
server R2 argocd-redis-ha-announce-2:26379 check inter 1s
# Check Sentinel and whether they are nominated master
backend check_if_redis_is_master_1
mode tcp
option tcp-check
tcp-check connect
tcp-check send PING\r\n
tcp-check expect string +PONG
tcp-check send SENTINEL\ get-master-addr-by-name\ argocd\r\n
tcp-check expect string REPLACE_ANNOUNCE1
tcp-check send QUIT\r\n
server R0 argocd-redis-ha-announce-0:26379 check inter 1s
server R1 argocd-redis-ha-announce-1:26379 check inter 1s
server R2 argocd-redis-ha-announce-2:26379 check inter 1s
# Check Sentinel and whether they are nominated master
backend check_if_redis_is_master_2
mode tcp
option tcp-check
tcp-check connect
tcp-check send PING\r\n
tcp-check expect string +PONG
tcp-check send SENTINEL\ get-master-addr-by-name\ argocd\r\n
tcp-check expect string REPLACE_ANNOUNCE2
tcp-check send QUIT\r\n
server R0 argocd-redis-ha-announce-0:26379 check inter 1s
server R1 argocd-redis-ha-announce-1:26379 check inter 1s
server R2 argocd-redis-ha-announce-2:26379 check inter 1s
# decide redis backend to use
#master
frontend ft_redis_master
bind [::]:6379 v4v6
use_backend bk_redis_master
# Check all redis servers to see if they think they are master
backend bk_redis_master
mode tcp
option tcp-check
tcp-check connect
tcp-check send PING\r\n
tcp-check expect string +PONG
tcp-check send info\ replication\r\n
tcp-check expect string role:master
tcp-check send QUIT\r\n
tcp-check expect string +OK
use-server R0 if { srv_is_up(R0) } { nbsrv(check_if_redis_is_master_0) ge 2 }
server R0 argocd-redis-ha-announce-0:6379 check inter 1s fall 1 rise 1
use-server R1 if { srv_is_up(R1) } { nbsrv(check_if_redis_is_master_1) ge 2 }
server R1 argocd-redis-ha-announce-1:6379 check inter 1s fall 1 rise 1
use-server R2 if { srv_is_up(R2) } { nbsrv(check_if_redis_is_master_2) ge 2 }
server R2 argocd-redis-ha-announce-2:6379 check inter 1s fall 1 rise 1
frontend stats
mode http
bind [::]:9101 v4v6
http-request use-service prometheus-exporter if { path /metrics }
stats enable
stats uri /stats
stats refresh 10s
haproxy_init.sh: |
HAPROXY_CONF=/data/haproxy.cfg
cp /readonly/haproxy.cfg "$HAPROXY_CONF"
for loop in $(seq 1 10); do
getent hosts argocd-redis-ha-announce-0 && break
echo "Waiting for service argocd-redis-ha-announce-0 to be ready ($loop) ..." && sleep 1
done
ANNOUNCE_IP0=$(getent hosts "argocd-redis-ha-announce-0" | awk '{ print $1 }')
if [ -z "$ANNOUNCE_IP0" ]; then
echo "Could not resolve the announce ip for argocd-redis-ha-announce-0"
exit 1
fi
sed -i "s/REPLACE_ANNOUNCE0/$ANNOUNCE_IP0/" "$HAPROXY_CONF"
for loop in $(seq 1 10); do
getent hosts argocd-redis-ha-announce-1 && break
echo "Waiting for service argocd-redis-ha-announce-1 to be ready ($loop) ..." && sleep 1
done
ANNOUNCE_IP1=$(getent hosts "argocd-redis-ha-announce-1" | awk '{ print $1 }')
if [ -z "$ANNOUNCE_IP1" ]; then
echo "Could not resolve the announce ip for argocd-redis-ha-announce-1"
exit 1
fi
sed -i "s/REPLACE_ANNOUNCE1/$ANNOUNCE_IP1/" "$HAPROXY_CONF"
for loop in $(seq 1 10); do
getent hosts argocd-redis-ha-announce-2 && break
echo "Waiting for service argocd-redis-ha-announce-2 to be ready ($loop) ..." && sleep 1
done
ANNOUNCE_IP2=$(getent hosts "argocd-redis-ha-announce-2" | awk '{ print $1 }')
if [ -z "$ANNOUNCE_IP2" ]; then
echo "Could not resolve the announce ip for argocd-redis-ha-announce-2"
exit 1
fi
sed -i "s/REPLACE_ANNOUNCE2/$ANNOUNCE_IP2/" "$HAPROXY_CONF"
trigger-failover-if-master.sh: |
get_redis_role() {
is_master=$(
redis-cli \
-h localhost \
-p 6379 \
info | grep -c 'role:master' || true
)
}
get_redis_role
if [[ "$is_master" -eq 1 ]]; then
echo "This node is currently master, we trigger a failover."
response=$(
redis-cli \
-h localhost \
-p 26379 \
SENTINEL failover argocd
)
if [[ "$response" != "OK" ]] ; then
echo "$response"
exit 1
fi
timeout=30
while [[ "$is_master" -eq 1 && $timeout -gt 0 ]]; do
sleep 1
get_redis_role
timeout=$((timeout - 1))
done
echo "Failover successful"
fi