feat: refactor and improve steps

.gitea/workflows/render-manifests-dispatch.yaml

@@ -43,24 +43,39 @@ jobs:
           method: kubeconfig
           kubeconfig: ${{ secrets.KUBECONFIG }}
 
+      - name: Cache Helm Dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/helm
+            ~/.config/helm
+          key: helm-cache-${{ runner.os }}-${{ hashFiles('infrastructure/clusters/cl01tl/helm/**/Chart.yaml', 'infrastructure/clusters/cl01tl/helm/**/Chart.lock') }}
+          restore-keys: |
+            helm-cache-${{ runner.os }}-
+
       - name: Prepare Manifest Branch
         run: |
-          cd ${MANIFEST_DIR}
+          cd "${MANIFEST_DIR}"
 
+          echo ""
           echo ">> Configure git to use gitea-bot as user ..."
           git config user.name "gitea-bot"
           git config user.email "gitea-bot@alexlebens.net"
 
+          echo ""
           echo ">> Checking if PR branch exists ..."
-          if [[ $(git ls-remote --heads origin "${BRANCH_NAME}" | wc -l) -gt 0 ]]; then
+          if git ls-remote --exit-code --heads origin "${BRANCH_NAME}" > /dev/null 2>&1; then
+            echo ""
             echo ">> Branch '${BRANCH_NAME}' exists, pulling changes ..."
             git fetch origin "${BRANCH_NAME}"
             git checkout "${BRANCH_NAME}"
             git pull --rebase
 
           else
+            echo ""
             echo ">> Branch '${BRANCH_NAME}' does not exist, creating ..."
-            git checkout -b $BRANCH_NAME
+            git checkout -b "${BRANCH_NAME}"
+
           fi
 
           echo "----"
@@ -68,25 +83,29 @@ jobs:
       - name: Check which Directories have Changes
         id: check-dir-changes
         run: |
-          cd ${MAIN_DIR}
+          cd "${MAIN_DIR}"
-
-          RENDER_DIR=()
 
+          echo ""
           echo ">> Triggered on dispatch, will check all paths ..."
-          RENDER_DIR+=$(ls clusters/cl01tl/helm/)
+
+          # Extract names of charts
+          RENDER_DIR=$(find "clusters/${CLUSTER}/helm" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort -u)
 
           if [ -n "${RENDER_DIR}" ]; then
+            echo ""
             echo ">> Directories to Render:"
-            echo "$(echo "${RENDER_DIR}" | sort -u)"
+            echo "${RENDER_DIR}"
-
             echo "----"
 
-            echo "changes-detected=true" >> $GITEA_OUTPUT
+            echo "changes-detected=true" >> "$GITEA_OUTPUT"
-            echo "render-dir<<EOF" >> $GITEA_OUTPUT
+            echo "render-dir<<EOF" >> "$GITEA_OUTPUT"
-            echo "$(echo "${RENDER_DIR}" | sort -u)" >> $GITEA_OUTPUT
+            echo "${RENDER_DIR}" >> "$GITEA_OUTPUT"
-            echo "EOF" >> $GITEA_OUTPUT
+            echo "EOF" >> "$GITEA_OUTPUT"
+
           else
-            echo "changes-detected=false" >> $GITEA_OUTPUT
+            echo ">> No directories found"
+            echo "changes-detected=false" >> "$GITEA_OUTPUT"
+
           fi
 
       - name: Add Repositories
@@ -94,29 +113,54 @@ jobs:
         env:
           RENDER_DIR: ${{ steps.check-dir-changes.outputs.render-dir }}
         run: |
-          cd ${MAIN_DIR}
+          cd "${MAIN_DIR}"
 
+          echo ""
           echo ">> Adding repositories for chart dependencies ..."
-          for dir in ${RENDER_DIR}; do
+          for DIR in ${RENDER_DIR}; do
-            helm dependency list --max-col-width 120 ${MAIN_DIR}/clusters/${CLUSTER}/helm/$dir 2> /dev/null \
+            helm dependency list --max-col-width 120 "${MAIN_DIR}/clusters/${CLUSTER}/helm/${DIR}" 2> /dev/null \
-              | tail +2 | head -n -1 \
+              | tail -n +2 \
-              | awk '{ print "helm repo add " $1 " " $3 }' \
+              | awk 'NF > 0 { print $1, $3 }' \
-              | while read cmd; do
+              | while read -r REPO_NAME REPO_URL; do
-                if [[ "$cmd" == "*oci://*" ]]; then
+                if [[ "${REPO_URL}" == oci://* ]]; then
-                  echo ">> Ignoring OCI repo"
+                  echo ""
-                else
+                  echo ">> Ignoring OCI repo: ${REPO_URL}"
-                  echo "$cmd" | sh;
+
+                elif [[ -n "${REPO_NAME}" && -n "${REPO_URL}" ]]; then
+                  helm repo add "${REPO_NAME}" "${REPO_URL}"
+
                 fi
               done || true
           done
 
-          if helm repo list | tail +2 | read -r; then
+          if helm repo list > /dev/null 2>&1; then
+            echo ""
             echo ">> Update repository cache ..."
             helm repo update
+
           fi
 
           echo "----"
 
+      - name: Remove Changed Manifest Files
+        if: steps.check-dir-changes.outputs.changes-detected == 'true'
+        env:
+          RENDER_DIR: ${{ steps.check-dir-changes.outputs.render-dir }}
+        run: |
+          cd "${MANIFEST_DIR}"
+
+          echo ""
+          echo ">> Remove manfiest files and rebuild from source ..."
+
+          for DIR in ${RENDER_DIR}; do
+            local CHART_PATH=${MANIFEST_DIR}/clusters/${CLUSTER}/manifests/${DIR}
+
+            echo "${CHART_PATH}"
+            rm -rf ${CHART_PATH}/*
+          done
+
+          echo "----"
+
       - name: Render Helm Manifests
         id: render-manifests
         if: steps.check-dir-changes.outputs.changes-detected == 'true'
@@ -125,76 +169,81 @@ jobs:
         run: |
           cd ${MAIN_DIR}
 
+          echo ""
           echo ">> Rendering Manifests ..."
 
-          for dir in ${RENDER_DIR}; do
+          render_chart() {
-            chart_path=${MAIN_DIR}/clusters/${CLUSTER}/helm/$dir
+            local DIR="$1"
-            chart_name=$(basename "$chart_path")
+            local CHART_PATH="${MAIN_DIR}/clusters/${CLUSTER}/helm/${DIR}"
+            local CHART_NAME=$(basename "${CHART_PATH}")
 
             echo ""
-            echo ""
+            echo ">> Rendering ..."
-            echo ">> Rendering chart: $chart_name"
+            echo ">> Chart: ${CHART_NAME}"
-            echo ">> Chart path $chart_path"
+            echo ">> Path: ${CHART_PATH}"
 
-            if [ -f "$chart_path/Chart.yaml" ]; then
+            if [ -f "${CHART_PATH}/Chart.yaml" ]; then
-              OUTPUT_FOLDER="${MANIFEST_DIR}/clusters/${CLUSTER}/manifests/$chart_name/"
+              local OUTPUT_FOLDER="${MANIFEST_DIR}/clusters/${CLUSTER}/manifests/${CHART_NAME}"
-              TEMPLATE=""
 
-              mkdir -p ${MANIFEST_DIR}/clusters/${CLUSTER}/manifests/$chart_name
+              mkdir -p "${OUTPUT_FOLDER}"
-
+              cd "${CHART_PATH}"
-              cd $chart_path
 
               echo ""
-              echo ">> Updating helm dependency ..."
+              echo ">> Updating helm dependencies ..."
-              helm dependency update --skip-refresh
+              helm dependency update --skip-refresh > /dev/null
 
               echo ""
-              echo ">> Building helm dependency ..."
+              echo ">> Linting helm chart ..."
-              helm dependency build --skip-refresh
+              helm lint --namespace "${CHART_NAME}" --quiet
 
-              echo ""
+              local NAMESPACE="${CHART_NAME}"
-              echo ">> Linting helm ..."
+              case "${CHART_NAME}" in
-              helm lint --namespace "$chart_name"
-
-              echo ""
-              echo ">> Rendering templates ..."
-              case "$chart_name" in
                 "stack")
+                  NAMESPACE="argocd"
                   echo ""
-                  echo ">> Special Rendering for stack into argocd namespace ..."
+                  echo ">> Special Rendering into 'argocd' namespace ..."
-                  TEMPLATE=$(helm template $chart_name ./ --namespace argocd --include-crds --dry-run=server --api-versions "gateway.networking.k8s.io/v1/HTTPRoute")
                   ;;
                 "cilium" | "coredns" | "metrics-server" | "prometheus-operator-crds")
+                  NAMESPACE="kube-system"
                   echo ""
-                  echo ">> Special Rendering for $chart_name into kube-system namespace ..."
+                  echo ">> Special Rendering for ${CHART_NAME} into 'kube-system' namespace ..."
-                  TEMPLATE=$(helm template $chart_name ./ --namespace kube-system --include-crds --dry-run=server --api-versions "gateway.networking.k8s.io/v1/HTTPRoute")
                   ;;
                 *)
                   echo ""
-                  echo ">> Standard Rendering for $chart_name ..."
+                  echo ">> Standard Rendering for ${CHART_NAME} ..."
-                  TEMPLATE=$(helm template "$chart_name" ./ --namespace "$chart_name" --include-crds --dry-run=server --api-versions "gateway.networking.k8s.io/v1/HTTPRoute")
-                  ;;
               esac
 
               echo ""
               echo ">> Formating rendered template ..."
-              echo "$TEMPLATE" | yq '... comments=""' | yq 'select(. != null)' | yq -s '"'"$OUTPUT_FOLDER"'" + .kind + "-" + .metadata.name + ".yaml"'
+              local TEMPLATE
+              TEMPLATE=$(helm template "${CHART_NAME}" ./ --namespace "${NAMESPACE}" --include-crds)
+
+              # Format and split rendered template
+              echo "${TEMPLATE}" | yq '... comments=""' | yq 'select(. != null) | yq -s '"'"${OUTPUT_FOLDER}"'/" + .kind + "-" + .metadata.name + ".yaml"'
 
               # Strip comments again to ensure formatting correctness
-              for file in "$OUTPUT_FOLDER"/*; do
+              if ls "${OUTPUT_FOLDER}"/*.yaml 1> /dev/null 2>&1; then
-                yq -i '... comments=""' $file
+                yq -i '... comments=""' "${OUTPUT_FOLDER}"/*.yaml
-              done
+              fi
 
               echo ""
-              echo ">> Manifests for $chart_name rendered to $OUTPUT_FOLDER"
+              echo ">> Manifests for ${CHART_NAME} rendered to ${OUTPUT_FOLDER}:"
               ls $OUTPUT_FOLDER
               echo ""
             else
               echo ""
-              echo ">> Directory $chart_path does not contain a Chart.yaml. Skipping ..."
+              echo ">> Directory ${CHART_PATH} does not contain a Chart.yaml. Skipping ..."
               echo ""
             fi
-          done
+          }
+
+          export -f render_chart
+          export MAIN_DIR CLUSTER MANIFEST_DIR
+
+          # Run rendering in parallel
+          for DIR in ${RENDER_DIR}; do
+            echo "${DIR}"
+          done | xargs -n 1 -P 4 -I {} bash -c 'render_chart "$@"' _ {}
 
           echo "----"
 
@@ -202,40 +251,46 @@ jobs:
         id: check-changes
         if: steps.check-dir-changes.outputs.changes-detected == 'true'
         run: |
-          cd ${MANIFEST_DIR}
+          cd "${MANIFEST_DIR}"
 
           GIT_CHANGES=$(git status --porcelain)
 
-          if [ -n "$GIT_CHANGES" ]; then
+          if [ -n "${GIT_CHANGES}" ]; then
+            echo ""
             echo ">> Changes detected"
             git status --porcelain
             echo "changes-detected=true" >> $GITEA_OUTPUT
 
           else
+            echo ""
             echo ">> No changes detected, skipping PR creation"
 
           fi
 
           echo "----"
+          echo "----"
 
       - name: Commit and Push Changes
         id: commit-push
         if: steps.check-changes.outputs.changes-detected == 'true'
         run: |
-          cd ${MANIFEST_DIR}
+          cd "${MANIFEST_DIR}"
 
+          echo ""
           echo ">> Commiting changes to ${BRANCH_NAME} ..."
           git add .
           git commit -m "chore: Update manifests after change"
 
           REPO_URL="${{ secrets.REPO_URL }}/${{ gitea.repository }}"
-          echo ">> Pushing changes to $REPO_URL ..."
+          echo ""
-          git push -u "https://oauth2:${{ secrets.BOT_TOKEN }}@$(echo $REPO_URL | sed -e 's|https://||')" ${BRANCH_NAME}
+          echo ">> Pushing changes to ${REPO_URL} ..."
+
+          git push -u "https://oauth2:${{ secrets.BOT_TOKEN }}@${REPO_URL#*://}" "${BRANCH_NAME}"
 
           echo "----"
 
-          echo "HEAD_BRANCH=${BRANCH_NAME}" >> $GITEA_OUTPUT
+          echo "HEAD_BRANCH=${BRANCH_NAME}" >> "$GITEA_OUTPUT"
-          echo "push=true" >> $GITEA_OUTPUT
+          echo "push=true" >> "$GITEA_OUTPUT"
 
       - name: Check for Pull Request
         id: check-for-pull-requst