--- # Source: argocd/charts/argo-cd/charts/redis-ha/templates/redis-ha-configmap.yaml apiVersion: v1 kind: ConfigMap metadata: name: argocd-redis-ha-configmap namespace: "argocd" labels: heritage: Helm release: argocd chart: redis-ha-4.34.11 app: argocd-redis-ha data: redis.conf: | dir "/data" port 6379 rename-command FLUSHDB "" rename-command FLUSHALL "" maxmemory 0 maxmemory-policy volatile-lru min-replicas-max-lag 5 min-replicas-to-write 1 rdbchecksum yes rdbcompression yes repl-diskless-sync yes save "" sentinel.conf: | dir "/data" port 26379 sentinel down-after-milliseconds argocd 10000 sentinel failover-timeout argocd 180000 maxclients 10000 sentinel parallel-syncs argocd 5 init.sh: | echo "$(date) Start..." HOSTNAME="$(hostname)" INDEX="${HOSTNAME##*-}" SENTINEL_PORT=26379 ANNOUNCE_IP='' MASTER='' MASTER_GROUP="argocd" QUORUM="2" REDIS_CONF=/data/conf/redis.conf REDIS_PORT=6379 REDIS_TLS_PORT= SENTINEL_CONF=/data/conf/sentinel.conf SENTINEL_TLS_PORT= SERVICE=argocd-redis-ha SENTINEL_TLS_REPLICATION_ENABLED=false REDIS_TLS_REPLICATION_ENABLED=false set -eu sentinel_get_master() { set +e if [ "$SENTINEL_PORT" -eq 0 ]; then redis-cli -h "${SERVICE}" -p "${SENTINEL_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key sentinel get-master-addr-by-name "${MASTER_GROUP}" |\ grep -E '((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?s*$))' else redis-cli -h "${SERVICE}" -p "${SENTINEL_PORT}" sentinel get-master-addr-by-name "${MASTER_GROUP}" |\ grep -E '((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?s*$))' fi set -e } sentinel_get_master_retry() { master='' retry=${1} sleep=3 for i in $(seq 1 "${retry}"); do master=$(sentinel_get_master) if [ -n "${master}" ]; then break fi sleep $((sleep + i)) done echo "${master}" } identify_master() { echo "Identifying redis master (get-master-addr-by-name).." echo " using sentinel (argocd-redis-ha), sentinel group name (argocd)" MASTER="$(sentinel_get_master_retry 3)" if [ -n "${MASTER}" ]; then echo " $(date) Found redis master (${MASTER})" else echo " $(date) Did not find redis master (${MASTER})" fi } sentinel_update() { echo "Updating sentinel config.." echo " evaluating sentinel id (\${SENTINEL_ID_${INDEX}})" eval MY_SENTINEL_ID="\$SENTINEL_ID_${INDEX}" echo " sentinel id (${MY_SENTINEL_ID}), sentinel grp (${MASTER_GROUP}), quorum (${QUORUM})" sed -i "1s/^/sentinel myid ${MY_SENTINEL_ID}\\n/" "${SENTINEL_CONF}" if [ "$SENTINEL_TLS_REPLICATION_ENABLED" = true ]; then echo " redis master (${1}:${REDIS_TLS_PORT})" sed -i "2s/^/sentinel monitor ${MASTER_GROUP} ${1} ${REDIS_TLS_PORT} ${QUORUM} \\n/" "${SENTINEL_CONF}" else echo " redis master (${1}:${REDIS_PORT})" sed -i "2s/^/sentinel monitor ${MASTER_GROUP} ${1} ${REDIS_PORT} ${QUORUM} \\n/" "${SENTINEL_CONF}" fi echo "sentinel announce-ip ${ANNOUNCE_IP}" >> ${SENTINEL_CONF} if [ "$SENTINEL_PORT" -eq 0 ]; then echo " announce (${ANNOUNCE_IP}:${SENTINEL_TLS_PORT})" echo "sentinel announce-port ${SENTINEL_TLS_PORT}" >> ${SENTINEL_CONF} else echo " announce (${ANNOUNCE_IP}:${SENTINEL_PORT})" echo "sentinel announce-port ${SENTINEL_PORT}" >> ${SENTINEL_CONF} fi } redis_update() { echo "Updating redis config.." if [ "$REDIS_TLS_REPLICATION_ENABLED" = true ]; then echo " we are slave of redis master (${1}:${REDIS_TLS_PORT})" echo "slaveof ${1} ${REDIS_TLS_PORT}" >> "${REDIS_CONF}" echo "slave-announce-port ${REDIS_TLS_PORT}" >> ${REDIS_CONF} else echo " we are slave of redis master (${1}:${REDIS_PORT})" echo "slaveof ${1} ${REDIS_PORT}" >> "${REDIS_CONF}" echo "slave-announce-port ${REDIS_PORT}" >> ${REDIS_CONF} fi echo "slave-announce-ip ${ANNOUNCE_IP}" >> ${REDIS_CONF} } copy_config() { echo "Copying default redis config.." echo " to '${REDIS_CONF}'" cp /readonly-config/redis.conf "${REDIS_CONF}" echo "Copying default sentinel config.." echo " to '${SENTINEL_CONF}'" cp /readonly-config/sentinel.conf "${SENTINEL_CONF}" } setup_defaults() { echo "Setting up defaults.." echo " using statefulset index (${INDEX})" if [ "${INDEX}" = "0" ]; then echo "Setting this pod as master for redis and sentinel.." echo " using announce (${ANNOUNCE_IP})" redis_update "${ANNOUNCE_IP}" sentinel_update "${ANNOUNCE_IP}" echo " make sure ${ANNOUNCE_IP} is not a slave (slaveof no one)" sed -i "s/^.*slaveof.*//" "${REDIS_CONF}" else echo "Getting redis master ip.." echo " blindly assuming (${SERVICE}-announce-0) or (${SERVICE}-server-0) are master" DEFAULT_MASTER="$(getent_hosts 0 | awk '{ print $1 }')" if [ -z "${DEFAULT_MASTER}" ]; then echo "Error: Unable to resolve redis master (getent hosts)." exit 1 fi echo " identified redis (may be redis master) ip (${DEFAULT_MASTER})" echo "Setting default slave config for redis and sentinel.." echo " using master ip (${DEFAULT_MASTER})" redis_update "${DEFAULT_MASTER}" sentinel_update "${DEFAULT_MASTER}" fi } redis_ping() { set +e if [ "$REDIS_PORT" -eq 0 ]; then redis-cli -h "${MASTER}" -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key ping else redis-cli -h "${MASTER}" -p "${REDIS_PORT}" ping fi set -e } redis_ping_retry() { ping='' retry=${1} sleep=3 for i in $(seq 1 "${retry}"); do if [ "$(redis_ping)" = "PONG" ]; then ping='PONG' break fi sleep $((sleep + i)) MASTER=$(sentinel_get_master) done echo "${ping}" } find_master() { echo "Verifying redis master.." if [ "$REDIS_PORT" -eq 0 ]; then echo " ping (${MASTER}:${REDIS_TLS_PORT})" else echo " ping (${MASTER}:${REDIS_PORT})" fi if [ "$(redis_ping_retry 3)" != "PONG" ]; then echo " $(date) Can't ping redis master (${MASTER})" echo "Attempting to force failover (sentinel failover).." if [ "$SENTINEL_PORT" -eq 0 ]; then echo " on sentinel (${SERVICE}:${SENTINEL_TLS_PORT}), sentinel grp (${MASTER_GROUP})" if redis-cli -h "${SERVICE}" -p "${SENTINEL_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key sentinel failover "${MASTER_GROUP}" | grep -q 'NOGOODSLAVE' ; then echo " $(date) Failover returned with 'NOGOODSLAVE'" echo "Setting defaults for this pod.." setup_defaults return 0 fi else echo " on sentinel (${SERVICE}:${SENTINEL_PORT}), sentinel grp (${MASTER_GROUP})" if redis-cli -h "${SERVICE}" -p "${SENTINEL_PORT}" sentinel failover "${MASTER_GROUP}" | grep -q 'NOGOODSLAVE' ; then echo " $(date) Failover returned with 'NOGOODSLAVE'" echo "Setting defaults for this pod.." setup_defaults return 0 fi fi echo "Hold on for 10sec" sleep 10 echo "We should get redis master's ip now. Asking (get-master-addr-by-name).." if [ "$SENTINEL_PORT" -eq 0 ]; then echo " sentinel (${SERVICE}:${SENTINEL_TLS_PORT}), sentinel grp (${MASTER_GROUP})" else echo " sentinel (${SERVICE}:${SENTINEL_PORT}), sentinel grp (${MASTER_GROUP})" fi MASTER="$(sentinel_get_master)" if [ "${MASTER}" ]; then echo " $(date) Found redis master (${MASTER})" echo "Updating redis and sentinel config.." sentinel_update "${MASTER}" redis_update "${MASTER}" else echo "$(date) Error: Could not failover, exiting..." exit 1 fi else echo " $(date) Found reachable redis master (${MASTER})" echo "Updating redis and sentinel config.." sentinel_update "${MASTER}" redis_update "${MASTER}" fi } redis_ro_update() { echo "Updating read-only redis config.." echo " redis.conf set 'replica-priority 0'" echo "replica-priority 0" >> ${REDIS_CONF} } getent_hosts() { index=${1:-${INDEX}} service="${SERVICE}-announce-${index}" host=$(getent hosts "${service}") echo "${host}" } identify_announce_ip() { echo "Identify announce ip for this pod.." echo " using (${SERVICE}-announce-${INDEX}) or (${SERVICE}-server-${INDEX})" ANNOUNCE_IP=$(getent_hosts | awk '{ print $1 }') echo " identified announce (${ANNOUNCE_IP})" } mkdir -p /data/conf/ echo "Initializing config.." copy_config # where is redis master identify_master identify_announce_ip if [ -z "${ANNOUNCE_IP}" ]; then "Error: Could not resolve the announce ip for this pod" exit 1 elif [ "${MASTER}" ]; then find_master else setup_defaults fi if [ "${AUTH:-}" ]; then echo "Setting redis auth values.." ESCAPED_AUTH=$(echo "${AUTH}" | sed -e 's/[\/&]/\\&/g'); sed -i "s/replace-default-auth/${ESCAPED_AUTH}/" "${REDIS_CONF}" "${SENTINEL_CONF}" fi if [ "${SENTINELAUTH:-}" ]; then echo "Setting sentinel auth values" ESCAPED_AUTH_SENTINEL=$(echo "$SENTINELAUTH" | sed -e 's/[\/&]/\\&/g'); sed -i "s/replace-default-sentinel-auth/${ESCAPED_AUTH_SENTINEL}/" "$SENTINEL_CONF" fi echo "$(date) Ready..." fix-split-brain.sh: | HOSTNAME="$(hostname)" INDEX="${HOSTNAME##*-}" SENTINEL_PORT=26379 ANNOUNCE_IP='' MASTER='' MASTER_GROUP="argocd" QUORUM="2" REDIS_CONF=/data/conf/redis.conf REDIS_PORT=6379 REDIS_TLS_PORT= SENTINEL_CONF=/data/conf/sentinel.conf SENTINEL_TLS_PORT= SERVICE=argocd-redis-ha SENTINEL_TLS_REPLICATION_ENABLED=false REDIS_TLS_REPLICATION_ENABLED=false ROLE='' REDIS_MASTER='' set -eu sentinel_get_master() { set +e if [ "$SENTINEL_PORT" -eq 0 ]; then redis-cli -h "${SERVICE}" -p "${SENTINEL_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key sentinel get-master-addr-by-name "${MASTER_GROUP}" |\ grep -E '((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?s*$))' else redis-cli -h "${SERVICE}" -p "${SENTINEL_PORT}" sentinel get-master-addr-by-name "${MASTER_GROUP}" |\ grep -E '((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?s*$))' fi set -e } sentinel_get_master_retry() { master='' retry=${1} sleep=3 for i in $(seq 1 "${retry}"); do master=$(sentinel_get_master) if [ -n "${master}" ]; then break fi sleep $((sleep + i)) done echo "${master}" } identify_master() { echo "Identifying redis master (get-master-addr-by-name).." echo " using sentinel (argocd-redis-ha), sentinel group name (argocd)" MASTER="$(sentinel_get_master_retry 3)" if [ -n "${MASTER}" ]; then echo " $(date) Found redis master (${MASTER})" else echo " $(date) Did not find redis master (${MASTER})" fi } sentinel_update() { echo "Updating sentinel config.." echo " evaluating sentinel id (\${SENTINEL_ID_${INDEX}})" eval MY_SENTINEL_ID="\$SENTINEL_ID_${INDEX}" echo " sentinel id (${MY_SENTINEL_ID}), sentinel grp (${MASTER_GROUP}), quorum (${QUORUM})" sed -i "1s/^/sentinel myid ${MY_SENTINEL_ID}\\n/" "${SENTINEL_CONF}" if [ "$SENTINEL_TLS_REPLICATION_ENABLED" = true ]; then echo " redis master (${1}:${REDIS_TLS_PORT})" sed -i "2s/^/sentinel monitor ${MASTER_GROUP} ${1} ${REDIS_TLS_PORT} ${QUORUM} \\n/" "${SENTINEL_CONF}" else echo " redis master (${1}:${REDIS_PORT})" sed -i "2s/^/sentinel monitor ${MASTER_GROUP} ${1} ${REDIS_PORT} ${QUORUM} \\n/" "${SENTINEL_CONF}" fi echo "sentinel announce-ip ${ANNOUNCE_IP}" >> ${SENTINEL_CONF} if [ "$SENTINEL_PORT" -eq 0 ]; then echo " announce (${ANNOUNCE_IP}:${SENTINEL_TLS_PORT})" echo "sentinel announce-port ${SENTINEL_TLS_PORT}" >> ${SENTINEL_CONF} else echo " announce (${ANNOUNCE_IP}:${SENTINEL_PORT})" echo "sentinel announce-port ${SENTINEL_PORT}" >> ${SENTINEL_CONF} fi } redis_update() { echo "Updating redis config.." if [ "$REDIS_TLS_REPLICATION_ENABLED" = true ]; then echo " we are slave of redis master (${1}:${REDIS_TLS_PORT})" echo "slaveof ${1} ${REDIS_TLS_PORT}" >> "${REDIS_CONF}" echo "slave-announce-port ${REDIS_TLS_PORT}" >> ${REDIS_CONF} else echo " we are slave of redis master (${1}:${REDIS_PORT})" echo "slaveof ${1} ${REDIS_PORT}" >> "${REDIS_CONF}" echo "slave-announce-port ${REDIS_PORT}" >> ${REDIS_CONF} fi echo "slave-announce-ip ${ANNOUNCE_IP}" >> ${REDIS_CONF} } copy_config() { echo "Copying default redis config.." echo " to '${REDIS_CONF}'" cp /readonly-config/redis.conf "${REDIS_CONF}" echo "Copying default sentinel config.." echo " to '${SENTINEL_CONF}'" cp /readonly-config/sentinel.conf "${SENTINEL_CONF}" } setup_defaults() { echo "Setting up defaults.." echo " using statefulset index (${INDEX})" if [ "${INDEX}" = "0" ]; then echo "Setting this pod as master for redis and sentinel.." echo " using announce (${ANNOUNCE_IP})" redis_update "${ANNOUNCE_IP}" sentinel_update "${ANNOUNCE_IP}" echo " make sure ${ANNOUNCE_IP} is not a slave (slaveof no one)" sed -i "s/^.*slaveof.*//" "${REDIS_CONF}" else echo "Getting redis master ip.." echo " blindly assuming (${SERVICE}-announce-0) or (${SERVICE}-server-0) are master" DEFAULT_MASTER="$(getent_hosts 0 | awk '{ print $1 }')" if [ -z "${DEFAULT_MASTER}" ]; then echo "Error: Unable to resolve redis master (getent hosts)." exit 1 fi echo " identified redis (may be redis master) ip (${DEFAULT_MASTER})" echo "Setting default slave config for redis and sentinel.." echo " using master ip (${DEFAULT_MASTER})" redis_update "${DEFAULT_MASTER}" sentinel_update "${DEFAULT_MASTER}" fi } redis_ping() { set +e if [ "$REDIS_PORT" -eq 0 ]; then redis-cli -h "${MASTER}" -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key ping else redis-cli -h "${MASTER}" -p "${REDIS_PORT}" ping fi set -e } redis_ping_retry() { ping='' retry=${1} sleep=3 for i in $(seq 1 "${retry}"); do if [ "$(redis_ping)" = "PONG" ]; then ping='PONG' break fi sleep $((sleep + i)) MASTER=$(sentinel_get_master) done echo "${ping}" } find_master() { echo "Verifying redis master.." if [ "$REDIS_PORT" -eq 0 ]; then echo " ping (${MASTER}:${REDIS_TLS_PORT})" else echo " ping (${MASTER}:${REDIS_PORT})" fi if [ "$(redis_ping_retry 3)" != "PONG" ]; then echo " $(date) Can't ping redis master (${MASTER})" echo "Attempting to force failover (sentinel failover).." if [ "$SENTINEL_PORT" -eq 0 ]; then echo " on sentinel (${SERVICE}:${SENTINEL_TLS_PORT}), sentinel grp (${MASTER_GROUP})" if redis-cli -h "${SERVICE}" -p "${SENTINEL_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key sentinel failover "${MASTER_GROUP}" | grep -q 'NOGOODSLAVE' ; then echo " $(date) Failover returned with 'NOGOODSLAVE'" echo "Setting defaults for this pod.." setup_defaults return 0 fi else echo " on sentinel (${SERVICE}:${SENTINEL_PORT}), sentinel grp (${MASTER_GROUP})" if redis-cli -h "${SERVICE}" -p "${SENTINEL_PORT}" sentinel failover "${MASTER_GROUP}" | grep -q 'NOGOODSLAVE' ; then echo " $(date) Failover returned with 'NOGOODSLAVE'" echo "Setting defaults for this pod.." setup_defaults return 0 fi fi echo "Hold on for 10sec" sleep 10 echo "We should get redis master's ip now. Asking (get-master-addr-by-name).." if [ "$SENTINEL_PORT" -eq 0 ]; then echo " sentinel (${SERVICE}:${SENTINEL_TLS_PORT}), sentinel grp (${MASTER_GROUP})" else echo " sentinel (${SERVICE}:${SENTINEL_PORT}), sentinel grp (${MASTER_GROUP})" fi MASTER="$(sentinel_get_master)" if [ "${MASTER}" ]; then echo " $(date) Found redis master (${MASTER})" echo "Updating redis and sentinel config.." sentinel_update "${MASTER}" redis_update "${MASTER}" else echo "$(date) Error: Could not failover, exiting..." exit 1 fi else echo " $(date) Found reachable redis master (${MASTER})" echo "Updating redis and sentinel config.." sentinel_update "${MASTER}" redis_update "${MASTER}" fi } redis_ro_update() { echo "Updating read-only redis config.." echo " redis.conf set 'replica-priority 0'" echo "replica-priority 0" >> ${REDIS_CONF} } getent_hosts() { index=${1:-${INDEX}} service="${SERVICE}-announce-${index}" host=$(getent hosts "${service}") echo "${host}" } identify_announce_ip() { echo "Identify announce ip for this pod.." echo " using (${SERVICE}-announce-${INDEX}) or (${SERVICE}-server-${INDEX})" ANNOUNCE_IP=$(getent_hosts | awk '{ print $1 }') echo " identified announce (${ANNOUNCE_IP})" } redis_role() { set +e if [ "$REDIS_PORT" -eq 0 ]; then ROLE=$(redis-cli -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key info | grep role | sed 's/role://' | sed 's/\r//') else ROLE=$(redis-cli -p "${REDIS_PORT}" info | grep role | sed 's/role://' | sed 's/\r//') fi set -e } identify_redis_master() { set +e if [ "$REDIS_PORT" -eq 0 ]; then REDIS_MASTER=$(redis-cli -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key info | grep master_host | sed 's/master_host://' | sed 's/\r//') else REDIS_MASTER=$(redis-cli -p "${REDIS_PORT}" info | grep master_host | sed 's/master_host://' | sed 's/\r//') fi set -e } reinit() { set +e sh /readonly-config/init.sh if [ "$REDIS_PORT" -eq 0 ]; then echo "shutdown" | redis-cli -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/ca.crt --cert /tls-certs/redis.crt --key /tls-certs/redis.key else echo "shutdown" | redis-cli -p "${REDIS_PORT}" fi set -e } identify_announce_ip while [ -z "${ANNOUNCE_IP}" ]; do echo "Error: Could not resolve the announce ip for this pod." sleep 30 identify_announce_ip done trap "exit 0" TERM while true; do sleep 60 # where is redis master identify_master if [ "$MASTER" = "$ANNOUNCE_IP" ]; then redis_role if [ "$ROLE" != "master" ]; then echo "waiting for redis to become master" sleep 10 identify_master redis_role echo "Redis role is $ROLE, expected role is master. No need to reinitialize." if [ "$ROLE" != "master" ]; then echo "Redis role is $ROLE, expected role is master, reinitializing" reinit fi fi elif [ "${MASTER}" ]; then identify_redis_master if [ "$REDIS_MASTER" != "$MASTER" ]; then echo "Redis master and local master are not the same. waiting." sleep 10 identify_master identify_redis_master echo "Redis master is ${MASTER}, expected master is ${REDIS_MASTER}. No need to reinitialize." if [ "${REDIS_MASTER}" != "${MASTER}" ]; then echo "Redis master is ${MASTER}, expected master is ${REDIS_MASTER}, reinitializing" reinit fi fi fi done haproxy.cfg: | defaults REDIS mode tcp timeout connect 4s timeout server 330s timeout client 330s timeout check 2s listen health_check_http_url bind [::]:8888 v4v6 mode http monitor-uri /healthz option dontlognull # Check Sentinel and whether they are nominated master backend check_if_redis_is_master_0 mode tcp option tcp-check tcp-check connect tcp-check send PING\r\n tcp-check expect string +PONG tcp-check send SENTINEL\ get-master-addr-by-name\ argocd\r\n tcp-check expect string REPLACE_ANNOUNCE0 tcp-check send QUIT\r\n server R0 argocd-redis-ha-announce-0:26379 check inter 1s server R1 argocd-redis-ha-announce-1:26379 check inter 1s server R2 argocd-redis-ha-announce-2:26379 check inter 1s # Check Sentinel and whether they are nominated master backend check_if_redis_is_master_1 mode tcp option tcp-check tcp-check connect tcp-check send PING\r\n tcp-check expect string +PONG tcp-check send SENTINEL\ get-master-addr-by-name\ argocd\r\n tcp-check expect string REPLACE_ANNOUNCE1 tcp-check send QUIT\r\n server R0 argocd-redis-ha-announce-0:26379 check inter 1s server R1 argocd-redis-ha-announce-1:26379 check inter 1s server R2 argocd-redis-ha-announce-2:26379 check inter 1s # Check Sentinel and whether they are nominated master backend check_if_redis_is_master_2 mode tcp option tcp-check tcp-check connect tcp-check send PING\r\n tcp-check expect string +PONG tcp-check send SENTINEL\ get-master-addr-by-name\ argocd\r\n tcp-check expect string REPLACE_ANNOUNCE2 tcp-check send QUIT\r\n server R0 argocd-redis-ha-announce-0:26379 check inter 1s server R1 argocd-redis-ha-announce-1:26379 check inter 1s server R2 argocd-redis-ha-announce-2:26379 check inter 1s # decide redis backend to use #master frontend ft_redis_master bind [::]:6379 v4v6 use_backend bk_redis_master # Check all redis servers to see if they think they are master backend bk_redis_master mode tcp option tcp-check tcp-check connect tcp-check send PING\r\n tcp-check expect string +PONG tcp-check send info\ replication\r\n tcp-check expect string role:master tcp-check send QUIT\r\n tcp-check expect string +OK use-server R0 if { srv_is_up(R0) } { nbsrv(check_if_redis_is_master_0) ge 2 } server R0 argocd-redis-ha-announce-0:6379 check inter 1s fall 1 rise 1 use-server R1 if { srv_is_up(R1) } { nbsrv(check_if_redis_is_master_1) ge 2 } server R1 argocd-redis-ha-announce-1:6379 check inter 1s fall 1 rise 1 use-server R2 if { srv_is_up(R2) } { nbsrv(check_if_redis_is_master_2) ge 2 } server R2 argocd-redis-ha-announce-2:6379 check inter 1s fall 1 rise 1 frontend stats mode http bind [::]:9101 v4v6 http-request use-service prometheus-exporter if { path /metrics } stats enable stats uri /stats stats refresh 10s haproxy_init.sh: | HAPROXY_CONF=/data/haproxy.cfg cp /readonly/haproxy.cfg "$HAPROXY_CONF" for loop in $(seq 1 10); do getent hosts argocd-redis-ha-announce-0 && break echo "Waiting for service argocd-redis-ha-announce-0 to be ready ($loop) ..." && sleep 1 done ANNOUNCE_IP0=$(getent hosts "argocd-redis-ha-announce-0" | awk '{ print $1 }') if [ -z "$ANNOUNCE_IP0" ]; then echo "Could not resolve the announce ip for argocd-redis-ha-announce-0" exit 1 fi sed -i "s/REPLACE_ANNOUNCE0/$ANNOUNCE_IP0/" "$HAPROXY_CONF" for loop in $(seq 1 10); do getent hosts argocd-redis-ha-announce-1 && break echo "Waiting for service argocd-redis-ha-announce-1 to be ready ($loop) ..." && sleep 1 done ANNOUNCE_IP1=$(getent hosts "argocd-redis-ha-announce-1" | awk '{ print $1 }') if [ -z "$ANNOUNCE_IP1" ]; then echo "Could not resolve the announce ip for argocd-redis-ha-announce-1" exit 1 fi sed -i "s/REPLACE_ANNOUNCE1/$ANNOUNCE_IP1/" "$HAPROXY_CONF" for loop in $(seq 1 10); do getent hosts argocd-redis-ha-announce-2 && break echo "Waiting for service argocd-redis-ha-announce-2 to be ready ($loop) ..." && sleep 1 done ANNOUNCE_IP2=$(getent hosts "argocd-redis-ha-announce-2" | awk '{ print $1 }') if [ -z "$ANNOUNCE_IP2" ]; then echo "Could not resolve the announce ip for argocd-redis-ha-announce-2" exit 1 fi sed -i "s/REPLACE_ANNOUNCE2/$ANNOUNCE_IP2/" "$HAPROXY_CONF" trigger-failover-if-master.sh: | get_redis_role() { is_master=$( redis-cli \ -h localhost \ -p 6379 \ info | grep -c 'role:master' || true ) } get_redis_role if [[ "$is_master" -eq 1 ]]; then echo "This node is currently master, we trigger a failover." response=$( redis-cli \ -h localhost \ -p 26379 \ SENTINEL failover argocd ) if [[ "$response" != "OK" ]] ; then echo "$response" exit 1 fi timeout=30 while [[ "$is_master" -eq 1 && $timeout -gt 0 ]]; do sleep 1 get_redis_role timeout=$((timeout - 1)) done echo "Failover successful" fi