Test branch 2024.04

CHANGELOG.md

@@ -1,5 +1,19 @@
 # SAS Viya Monitoring for Kubernetes
 
+## Unreleased
+* **Overall**
+
+* **Metrics**
+
+* **Logging**
+  * [FIX] Corrected parser definition for Consul messages to eliminate ERROR/WARNING messages in Fluent Bit pod logs
+  * [CHANGE] Added parser/processing for Redis log messsages
+  * [CHANGE] Added parser/processing for Calico (CNI) log messsages
+  * [UPGRADE] Upgraded OpenSearch/OpenSearch Dashboards from 2.10.0 to 2.12.0
+  * [UPGRADE] Elasticsearch Exporter has been upgraded from 1.6.0 to 1.7.0
+
+* **Tracing**
+
 ## Version 1.2.23 (19MAR2024)
 * **Overall**
   * [CHANGE] Drop support for OpenShift 4.11; the minimum supported version of OpenShift is now 4.12.
@@ -34,8 +48,6 @@ and `logging/user-values-es-exporter.yaml`.
   * [FIX] Corrected typo in `logging/bin/deploy_fluentbit_azmonitor.sh` that prevented the script from executing properly.
   * [UPGRADE] Fluent Bit has been upgraded from version 2.1.10 to 2.2.2
 
-* **Tracing**
-
 
 ## Version 1.2.22 (13FEB2024)
 * **Overall**

component_versions.env

@@ -11,8 +11,8 @@
 #Elasticsearch Exporter
 ESEXPORTER_HELM_CHART_REPO=prometheus-community
 ESEXPORTER_HELM_CHART_NAME=prometheus-elasticsearch-exporter
-ESEXPORTER_HELM_CHART_VERSION=5.3.1
-ES_EXPORTER_FULL_IMAGE="quay.io/prometheuscommunity/elasticsearch-exporter:v1.6.0"
+ESEXPORTER_HELM_CHART_VERSION=5.6.0
+ES_EXPORTER_FULL_IMAGE="quay.io/prometheuscommunity/elasticsearch-exporter:v1.7.0"
 
 #Fluent Bit
 FLUENTBIT_HELM_CHART_REPO=fluent
@@ -23,15 +23,15 @@ FB_FULL_IMAGE="cr.fluentbit.io/fluent/fluent-bit:2.2.2"
 #OpenSearch
 OPENSEARCH_HELM_CHART_REPO=opensearch
 OPENSEARCH_HELM_CHART_NAME=opensearch
-OPENSEARCH_HELM_CHART_VERSION=2.15.0
-OS_FULL_IMAGE="docker.io/opensearchproject/opensearch:2.10.0"
+OPENSEARCH_HELM_CHART_VERSION=2.18.0
+OS_FULL_IMAGE="docker.io/opensearchproject/opensearch:2.12.0"
 OS_SYSCTL_FULL_IMAGE="docker.io/library/busybox:latest"
 
 #OpenSearch Dashboards
 OSD_HELM_CHART_REPO=opensearch
 OSD_HELM_CHART_NAME=opensearch-dashboards
-OSD_HELM_CHART_VERSION=2.13.0
-OSD_FULL_IMAGE="docker.io/opensearchproject/opensearch-dashboards:2.10.0"
+OSD_HELM_CHART_VERSION=2.16.0
+OSD_FULL_IMAGE="docker.io/opensearchproject/opensearch-dashboards:2.12.0"
 
 #Grafana (when deployed on OpenShift)
 OPENSHIFT_GRAFANA_CHART_REPO=grafana

logging/fb/fluent-bit_config.configmap_opensearch.yaml

@@ -6,7 +6,7 @@ data:
     [FILTER]
        Name modify
        Match *
-       Set  fb_configMap_version 0.2.25
+       Set  fb_configMap_version 0.2.26J
        #Set clusterID NOT_SET
        # initialized to N to force level standardization
        Set  __temp_level_fixed   N
@@ -718,7 +718,24 @@ data:
        Condition       Key_value_matches  flagged4parsing s2operator
        Rename          source      PROPS__source
        Rename          component   PROPS__component
-
+    [FILTER]
+       #
+       # Redis
+       #
+       Alias           flag_redis_msgs
+       Name            modify
+       Match           kube.*
+       Condition       Key_value_matches   temp_k8sapp_name sas-redis-server|sas-airflow-redis
+       Copy            message   redis_message
+       Rename          message   unparsed_message
+       Add             flagged4parsing redis
+    [FILTER]
+       Alias           parse_redis_msgs
+       Name            parser
+       Match           kube.*
+       Key_name        redis_message
+       Parser          redis
+       Reserve_Data    On
     [FILTER]
        #
        # Zalando (Postgres)
@@ -776,6 +793,7 @@ data:
        Parser          misc6
        Parser          misc7
        Parser          misc8
+       Parser          calico
        Parser          collectapiT
        Parser          collectapi
        Parser          haproxy_default
@@ -835,7 +853,7 @@ data:
        Alias          fix_level_debug
        Name           modify
        Match          *
-       Condition      Key_value_matches level (?i:DEBUG)|(?:\AD\z)|(?i:hint)|(?i:fine)|(?i:DBG)|(?i:DETAIL)
+       Condition      Key_value_matches level (?i:DEBUG)|(?:\AD\z)|(?i:hint)|(?i:fine)|(?i:DBG)|(?i:DETAIL)|(\.)
        Set            level DEBUG
        Set            __temp_level_fixed   Y
     [FILTER]
@@ -862,7 +880,7 @@ data:
        Name           modify
        Match          *
        Condition      Key_value_does_not_equal __temp_level_fixed Y
-       Condition      Key_value_matches level (?i:inf\w*)|(?i:note\w*)|(?i:log\w*)|(?:\AI\z)|(?i:Normal)|(?i:notice)
+       Condition      Key_value_matches level (?i:inf\w*)|(?i:note\w*)|(?i:log\w*)|(?:\AI\z)|(?i:Normal)|(?i:notice)|(-)|(\*)
        Set            level INFO
        Set            __temp_level_fixed   Y
     [FILTER]
@@ -889,7 +907,7 @@ data:
        Name           modify
        Match          *
        Condition      Key_value_does_not_equal __temp_level_fixed Y
-       Condition      Key_value_matches level (?i:war\w*)|(?:\AW\z)|(?i:wrn\w*)
+       Condition      Key_value_matches level (?i:war\w*)|(?:\AW\z)|(?i:wrn\w*)|(#)
        Set            level WARNING
        Set            __temp_level_fixed   Y
     [FILTER]

logging/fb/viya-parsers.conf

@@ -1,4 +1,4 @@
-# Version 1.9.10
+# Version 1.9.11-snapshot3
 [PARSER]
        Name          consul
        Format        regex
@@ -11,7 +11,7 @@
        Format        regex
        Regex (?<timestamp>\d{4}-\d{2}-\d{2}T(\d{2}:?){3}\.\d{3})Z \[(?<level>\w+)\]\s+(?<message>.*)
        Time_Key      timestamp
-       Time_format   %Y-%m-%dT%H:%M:%S.%L%Z
+       Time_format   %Y-%m-%dT%H:%M:%S.%L
 [PARSER]
        Name          consul_telemetry
        Format        regex
@@ -344,3 +344,20 @@
        regex         (?<timestamp>\d{4}(-\d{2}){2} (\d{2}:?){3},\d{3})\s+-\s(?<message>(?<phase>\w+)\s+-\s(?<level>\w+)\s+-\s+.*)
        Time_Key      timestamp
        Time_format   %Y-%m-%d %H:%M:%S,%L
+[PARSER]
+       Name          redis
+       # Redis
+       # Example: 71:S 18 Mar 2024 14:14:11.269 * Background append only file rewriting started by pid 143
+       # NOTE: Redis log messages use special characters to indicate message level
+       Format        regex
+       Regex \S+\s(?<timestamp>\d\d?\s[A-Z][a-z]{2}\s\d{4}\s(\d\d:?){3}\.\d{3})\s(?<level>\S)\s(?<message>.+)
+       Time_Key      timestamp
+       Time_format   %d %b %Y %T.%L
+[PARSER]
+       Name          calico
+       # Calico
+       # Example: 2024-03-21 15:56:07.077 [INFO][78] felix/status_combiner.go 98: Reporting combined status. id=proto.WorkloadEndpointID{OrchestratorId:"k8s", WorkloadId:"d73310/sas-crunchy-platform-postgres-00-njfn-0", EndpointId:"eth0"} status="up"
+       Format        regex
+       Regex  (?<timestamp>\d{4}(-\d{2}){2} (\d{2}:?){3}\.\d{3})\s\[(?<level>[A-Z]+)\](?<message>.+)
+       Time_Key      timestamp
+       Time_format   %F% %T.%L