ADD: Metrics and Alerts for Solr

charts/solr/templates/alerts.yaml

@@ -0,0 +1,92 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ .Release.Name }}
+  namespace: {{ .Release.Namespace }}
+  generation: 1
+  labels:
+    app: kube-prometheus-stack
+    heritage: Helm
+    release: prometheus
+spec:
+  groups:
+    - name: {{ .Release.Namespace }}.{{ .Release.Name }}-solr.rules
+      rules:
+        - alert: SolrDown
+          expr: up{namespace="{{ .Release.Namespace }}", instance=~"{{ .Release.Name }}-solr-.*"} == 0
+          for: 0m
+          labels:
+            severity: critical
+            servicealert: "true"
+            namespace: {{ .Release.Namespace }}
+            service: {{ .Values.name }}
+          annotations:
+            summary: 'Solr instance {{ .Release.Name }} is down'
+            description: 'Solr instance {{ .Release.Name }} is down'
+
+        - alert: SolrHighHeapMemoryUsage
+          expr: |
+            solr_metrics_jvm_memory_heap_bytes{item="used", namespace="{{ .Release.Namespace }}", instance=~"{{ .Release.Name }}-solr-.*"}
+            / solr_metrics_jvm_memory_heap_bytes{item="max", namespace="{{ .Release.Namespace }}", instance=~"{{ .Release.Name }}-solr-.*"} * 100 > 80
+          for: 2m
+          labels:
+            severity: warning
+            servicealert: "true"
+            namespace: {{ .Release.Namespace }}
+            service: {{ .Values.name }}
+          annotations:
+            summary: 'Solr instance {{ .Release.Name }} high heap memory usage'
+            description: 'More than 80% of Solr heap memory is in use for {{ .Release.Name }}'
+
+        - alert: SolrHighCpuUsage
+          expr: |
+            solr_metrics_jvm_os_cpu_load{item="processCpuLoad", namespace="{{ .Release.Namespace }}", instance=~"{{ .Release.Name }}-solr-.*"} > 0.8
+          for: 2m
+          labels:
+            severity: warning
+            servicealert: "true"
+            namespace: {{ .Release.Namespace }}
+            service: {{ .Values.name }}
+          annotations:
+            summary: 'Solr instance {{ .Release.Name }} high CPU usage'
+            description: 'Solr instance {{ .Release.Name }} CPU load is > 80%'
+
+        - alert: SolrLowDiskSpace
+          expr: |
+            solr_metrics_node_core_root_fs_bytes{item="usableSpace", namespace="{{ .Release.Namespace }}", instance=~"{{ .Release.Name }}-solr-.*"}
+            / solr_metrics_node_core_root_fs_bytes{item="totalSpace", namespace="{{ .Release.Namespace }}", instance=~"{{ .Release.Name }}-solr-.*"} * 100 < 15
+          for: 2m
+          labels:
+            severity: warning
+            servicealert: "true"
+            namespace: {{ .Release.Namespace }}
+            service: {{ .Values.name }}
+          annotations:
+            summary: 'Solr instance {{ .Release.Name }} low disk space'
+            description: 'Solr instance {{ .Release.Name }} has less than 15% free disk space'
+
+        - alert: SolrServerErrors
+          expr: |
+            increase(solr_metrics_node_server_errors_total{namespace="{{ .Release.Namespace }}", instance=~"{{ .Release.Name }}-solr-.*"}[1m]) > 0
+          for: 2m
+          labels:
+            severity: warning
+            servicealert: "true"
+            namespace: {{ .Release.Namespace }}
+            service: {{ .Values.name }}
+          annotations:
+            summary: 'Solr instance {{ .Release.Name }} server errors'
+            description: 'Solr instance {{ .Release.Name }} is experiencing server errors'
+
+        - alert: SolrClientErrors
+          expr: |
+            increase(solr_metrics_node_client_errors_total{namespace="{{ .Release.Namespace }}", instance=~"{{ .Release.Name }}-solr-.*"}[1m]) > 10
+          for: 2m
+          labels:
+            severity: warning
+            servicealert: "true"
+            namespace: {{ .Release.Namespace }}
+            service: {{ .Values.name }}
+          annotations:
+            summary: 'Solr instance {{ .Release.Name }} client errors'
+            description: 'Solr instance {{ .Release.Name }} is experiencing > 10 client errors per minute'

charts/solr/templates/service.yaml

@@ -13,3 +13,8 @@ spec:
     - protocol: TCP
       port: 8983
       targetPort: 8983
+      name: solr
+    - protocol: TCP
+      port: 2121
+      targetPort: 2121
+      name: metrics

charts/solr/templates/serviceMonitor.yaml

@@ -0,0 +1,16 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ .Release.Name }}-solr
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: solr
+    release: prometheus
+spec:
+  selector:
+    matchLabels:
+      app: solr
+  endpoints:
+    - port: metrics
+      interval: 30s
+      path: /metrics

charts/solr/templates/statefulset.yaml

@@ -56,6 +56,38 @@ spec:
             - name: security-config
               mountPath: /var/solr/data/security.json
               subPath: security.json
+        - name: solr-exporter
+          image: "solr:{{ .Values.version }}"
+          command:
+            - "/opt/solr/contrib/prometheus-exporter/bin/solr-exporter"
+            - "-p"
+            - "2121"
+            - "-b"
+            - "http://localhost:8983/solr"
+            - "-config-file"
+            - "/opt/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml"
+          ports:
+            - containerPort: 2121
+              name: metrics
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: 200m
+              memory: 512Mi
+          livenessProbe:
+            httpGet:
+              path: /metrics
+              port: 2121
+            initialDelaySeconds: 30
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /metrics
+              port: 2121
+            initialDelaySeconds: 15
+            periodSeconds: 10
       volumes:
         - name: security-config
           configMap:

promql/solr.yaml

@@ -0,0 +1,62 @@
+queries:
+  "<=0.0.5":
+    solr_up:
+      query: >-
+        up{job='<job-name>', namespace='<namespace>', pod=~'<service-name>.*' }
+
+    cpu_load:
+      query: >-
+        solr_metrics_jvm_os_cpu_load{item="processCpuLoad", namespace='<application-name>', pod=~'<service-name>.*'}
+
+    open_file_descriptors:
+      query: >-
+        solr_metrics_jvm_os_file_descriptors{item="openFileDescriptorCount", namespace='<application-name>', pod=~'<service-name>.*'}
+
+    http_requests_per_second:
+      query: >-
+        sum by (pod) (
+          rate(solr_metrics_node_requests_total{namespace='<application-name>', pod=~'<service-name>.*'}[1m])
+        )
+
+    http_errors_per_second:
+      query: >-
+        sum by (pod) (
+          rate(solr_metrics_node_errors_total{namespace='<application-name>', pod=~'<service-name>.*'}[1m])
+        )
+
+    gc_time_seconds:
+      query: >-
+        rate(solr_metrics_jvm_gc_seconds_total{namespace='<application-name>', pod=~'<service-name>.*'}[1m])
+
+    gc_count:
+      query: >-
+        rate(solr_metrics_jvm_gc_total{namespace='<application-name>', pod=~'<service-name>.*'}[1m])
+
+    thread_count:
+      query: >-
+        solr_metrics_jvm_threads{item="count", namespace='<application-name>', pod=~'<service-name>.*'}
+
+    response_codes:
+      query: >-
+        sum by (pod, status) (
+          rate(solr_metrics_jetty_response_total{namespace='<application-name>', pod=~'<service-name>.*'}[1m])
+        )
+
+    request_methods:
+      query: >-
+        sum by (pod, method) (
+          rate(solr_metrics_jetty_requests_total{namespace='<application-name>', pod=~'<service-name>.*'}[1m])
+        )
+
+    request_handler_time:
+      query: >-
+        rate(solr_metrics_node_time_seconds_total{namespace='<application-name>', pod=~'<service-name>.*'}[1m])
+
+    loaded_cores:
+      query: >-
+        solr_metrics_node_cores{item="loaded", namespace='<application-name>', pod=~'<service-name>.*'}
+
+    memory_usage:
+      query: >-
+        solr_metrics_jvm_memory_bytes{item="used", namespace='<application-name>', pod=~'<service-name>.*'}
+      unit: "bytes"