enable metrics, improve failover docs, stop removing Redis commands, … (

#6) * enable metrics, improve failover docs, stop removing Redis commands, remove managed by helm annotation * remove annotation from sts template * add codeowners
utilitywarehouse · Dec 7, 2023 · d2e93a2 · d2e93a2
1 parent aa0dd18
commit d2e93a2
Show file tree

Hide file tree

Showing 6 changed files with 177 additions and 7 deletions.
diff --git a/CODEOWNERS b/CODEOWNERS
@@ -0,0 +1 @@
+* @utilitywarehouse/dev-enablement
diff --git a/redis/README.md b/redis/README.md
@@ -7,8 +7,11 @@ This manifest is build on the base of [Bitnami Sentinel Redis Helm chart](https:
 Before using those manifests, consider whether Sentinel is suitable for your use:
 
 - [Redis Sentinel](https://redis.io/docs/management/sentinel/) allows using multiple databases in one instance,
-however- it doesn't provide sharding. This means, nodes
-other than master are just failover replicas. This grants high availability.
+however- it doesn't provide sharding. This means, nodes other than master are just read-only replicas. <br>
+[It is possible to grant high availability by enabling Sentinel failover](https://github.com/bitnami/charts/tree/main/bitnami/redis#master-replicas-with-sentinel)- in this case, master failure would cause 
+election of the new master from replica nodes. However, this would demand client library querying the master address.
+
+
 
 - [Redis cluster](https://github.com/bitnami/charts/tree/main/bitnami/redis-cluster)
 (we don't have our manifests for it yet) is alternative to Sentinel- it allows just
@@ -26,5 +29,5 @@ NAME="redis-shared" # Name of your Redis instance
 OPSLEVEL_APP_DESCRIPTION="cache for opslevel-k8s-deployer" # Description of your Redis in OpsLevel
 OPSLEVEL_APP_TIER="tier_4" # Tier of your Redis in OpsLevel- see https://wiki.uw.systems/posts/ops-level-nz4v4ka0#h1u0u-app-uw-systems-tier
 REDIS_SECRET_NAME="redis" # Name of the secret with your Redis password. See secret created in directory `example`.
-REDIS_REPLICA_COUNT=1 # Amount of failover replicas 
+REDIS_REPLICA_COUNT=1 # Amount of read-only replicas 
 ```
diff --git a/redis/gen-yaml/clean-upstream-kustomize-template.yaml b/redis/gen-yaml/clean-upstream-kustomize-template.yaml
@@ -5,6 +5,18 @@ resources:
   - redis.yaml
 
 patches:
+  # Remove unnecessary `app.kubernetes.io/managed-by: Helm` annotation
+  - patch: |-
+      - op: remove
+        path: /metadata/labels/app.kubernetes.io~1managed-by
+    target: { } # All resources
+  - patch: |-
+      - op: remove
+        path: /spec/template/metadata/labels/app.kubernetes.io~1managed-by
+    target:
+      group: apps
+      version: v1
+      kind: StatefulSet
   # remove these empty affinity nodes, as in Kustomize 5.0.x kustomize generates them as strings with value "null" instead of just null, and they can not be applied.
   # added this issue in kustomize: https://github.com/kubernetes-sigs/kustomize/issues/5171
   - patch: |-

diff --git a/redis/gen-yaml/gen.sh b/redis/gen-yaml/gen.sh
@@ -16,6 +16,8 @@ helm template "${NAME}" bitnami/redis --version "${BITNAMI_REDIS_RELEASE}" \
   --set commonAnnotations."app\.uw\.systems\/repos"="https://github.com/utilitywarehouse/shared-kustomize-bases/tree/main/redis" \
   --set auth.existingSecret="${REDIS_SECRET_NAME}" \
   --set replica.replicaCount="${REDIS_REPLICA_COUNT}" \
+  --set metrics.enabled="true" \
+  --set master.disableCommands="" \
   --set master.resources.requests.cpu="500m" \
   --set master.resources.limits.cpu="1000m" \
   --set master.resources.requests.memory="1Gi" \

diff --git a/redis/manifests/dev-enablement/redis-shared/upstream/kustomization.yaml b/redis/manifests/dev-enablement/redis-shared/upstream/kustomization.yaml
@@ -5,6 +5,18 @@ resources:
   - redis.yaml
 
 patches:
+  # Remove unnecessary `app.kubernetes.io/managed-by: Helm` annotation
+  - patch: |-
+      - op: remove
+        path: /metadata/labels/app.kubernetes.io~1managed-by
+    target: { } # All resources
+  - patch: |-
+      - op: remove
+        path: /spec/template/metadata/labels/app.kubernetes.io~1managed-by
+    target:
+      group: apps
+      version: v1
+      kind: StatefulSet
   # remove these empty affinity nodes, as in Kustomize 5.0.x kustomize generates them as strings with value "null" instead of just null, and they can not be applied.
   # added this issue in kustomize: https://github.com/kubernetes-sigs/kustomize/issues/5171
   - patch: |-

diff --git a/redis/manifests/dev-enablement/redis-shared/upstream/redis.yaml b/redis/manifests/dev-enablement/redis-shared/upstream/redis.yaml
@@ -44,8 +44,6 @@ data:
   master.conf: |-
     dir /data
     # User-supplied master configuration:
-    rename-command FLUSHDB ""
-    rename-command FLUSHALL ""
     # End of master configuration
   replica.conf: |-
     dir /data
@@ -311,6 +309,34 @@ spec:
     app.kubernetes.io/name: redis-shared
     app.kubernetes.io/component: master
 ---
+# Source: redis/templates/metrics-svc.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis-shared-metrics
+  namespace: "dev-enablement"
+  labels:
+    app.kubernetes.io/instance: redis-shared
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: redis-shared
+    app.kubernetes.io/version: 7.2.2
+    helm.sh/chart: redis-18.1.6
+    app.kubernetes.io/component: metrics
+  annotations:
+    app.uw.systems/description: cache for opslevel-k8s-deployer
+    app.uw.systems/repos: https://github.com/utilitywarehouse/shared-kustomize-bases/tree/main/redis
+    app.uw.systems/tier: tier_4
+spec:
+  type: ClusterIP
+  ports:
+    - name: http-metrics
+      port: 9121
+      protocol: TCP
+      targetPort: metrics
+  selector:
+    app.kubernetes.io/instance: redis-shared
+    app.kubernetes.io/name: redis-shared
+---
 # Source: redis/templates/replicas/service.yaml
 apiVersion: v1
 kind: Service
@@ -379,10 +405,12 @@ spec:
         helm.sh/chart: redis-18.1.6
         app.kubernetes.io/component: master
       annotations:
-        checksum/configmap: 86bcc953bb473748a3d3dc60b7c11f34e60c93519234d4c37f42e22ada559d47
+        checksum/configmap: e888d0f18b05c7bd70a8c6ee0ee303f66b2775fd57390463254884461a8cdf6c
         checksum/health: aff24913d801436ea469d8d374b2ddb3ec4c43ee7ab24663d5f8ff1a1b6991a9
         checksum/scripts: aea3e551fdf75c0c71eaaf1c928fdefef87f72f941de0d5034d5d6499891d349
         checksum/secret: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a
+        prometheus.io/port: "9121"
+        prometheus.io/scrape: "true"
     spec:
 
       securityContext:
@@ -487,6 +515,61 @@ spec:
               mountPath: /opt/bitnami/redis/etc/
             - name: tmp
               mountPath: /tmp
+        - name: metrics
+          image: docker.io/bitnami/redis-exporter:1.55.0-debian-11-r0
+          imagePullPolicy: "IfNotPresent"
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+              - ALL
+            runAsGroup: 0
+            runAsNonRoot: true
+            runAsUser: 1001
+            seccompProfile:
+              type: RuntimeDefault
+          command:
+            - /bin/bash
+            - -c
+            - |
+              if [[ -f '/secrets/redis-password' ]]; then
+              export REDIS_PASSWORD=$(cat /secrets/redis-password)
+              fi
+              redis_exporter
+          env:
+            - name: REDIS_ALIAS
+              value: redis-shared
+            - name: REDIS_USER
+              value: default
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: redis
+                  key: redis-password
+          ports:
+            - name: metrics
+              containerPort: 9121
+          livenessProbe:
+            failureThreshold: 5
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 5
+            tcpSocket:
+              port: metrics
+          readinessProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+            httpGet:
+              path: /
+              port: metrics
+          resources:
+            limits: {}
+            requests: {}
+          volumeMounts:
       volumes:
         - name: start-scripts
           configMap:
@@ -556,10 +639,12 @@ spec:
         helm.sh/chart: redis-18.1.6
         app.kubernetes.io/component: replica
       annotations:
-        checksum/configmap: 86bcc953bb473748a3d3dc60b7c11f34e60c93519234d4c37f42e22ada559d47
+        checksum/configmap: e888d0f18b05c7bd70a8c6ee0ee303f66b2775fd57390463254884461a8cdf6c
         checksum/health: aff24913d801436ea469d8d374b2ddb3ec4c43ee7ab24663d5f8ff1a1b6991a9
         checksum/scripts: aea3e551fdf75c0c71eaaf1c928fdefef87f72f941de0d5034d5d6499891d349
         checksum/secret: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a
+        prometheus.io/port: "9121"
+        prometheus.io/scrape: "true"
     spec:
 
       securityContext:
@@ -678,6 +763,61 @@ spec:
               mountPath: /opt/bitnami/redis/mounted-etc
             - name: redis-tmp-conf
               mountPath: /opt/bitnami/redis/etc
+        - name: metrics
+          image: docker.io/bitnami/redis-exporter:1.55.0-debian-11-r0
+          imagePullPolicy: "IfNotPresent"
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+              - ALL
+            runAsGroup: 0
+            runAsNonRoot: true
+            runAsUser: 1001
+            seccompProfile:
+              type: RuntimeDefault
+          command:
+            - /bin/bash
+            - -c
+            - |
+              if [[ -f '/secrets/redis-password' ]]; then
+              export REDIS_PASSWORD=$(cat /secrets/redis-password)
+              fi
+              redis_exporter
+          env:
+            - name: REDIS_ALIAS
+              value: redis-shared
+            - name: REDIS_USER
+              value: default
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: redis
+                  key: redis-password
+          ports:
+            - name: metrics
+              containerPort: 9121
+          livenessProbe:
+            failureThreshold: 5
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 5
+            tcpSocket:
+              port: metrics
+          readinessProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+            httpGet:
+              path: /
+              port: metrics
+          resources:
+            limits: {}
+            requests: {}
+          volumeMounts:
       volumes:
         - name: start-scripts
           configMap: