Add k8s label and annotations to pod logs (#937)

Signed-off-by: Pete Wall <[email protected]>

Author: petewall

charts/feature-pod-logs/README.md

@@ -36,11 +36,21 @@ Actual integration testing in a live environment should be done in the main [k8s
 
 ## Values
 
-### Logs Scrape: Pod Logs
+### Log Processing
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| annotations | object | `{"job":"k8s.grafana.com/logs.job"}` | Log labels to set with values copied from the Kubernetes Pod annotations. Format: `<log_label>: <kubernetes_annotation>`. |
+| extraLogProcessingStages | string | `""` | Stage blocks to be added to the loki.process component for pod logs. ([docs](https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#blocks)) This value is templated so that you can refer to other values from this file. |
+| labels | object | `{"app_kubernetes_io_name":"app.kubernetes.io/name"}` | Log labels to set with values copied from the Kubernetes Pod labels. Format: `<log_label>: <kubernetes_label>`. |
+
+### Pod Discovery
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | excludeNamespaces | list | `[]` | Do not capture logs from any pods in these namespaces. |
+| extraDiscoveryRules | string | `""` | Rules to filter pods for log gathering. Only used for "volumes" or "kubernetesApi" gather methods. |
+| gatherMethod | string | `"volumes"` | The method to gather pod logs. Options are "volumes", "kubernetesApi", "OpenShiftClusterLogForwarder" (experimental). |
 | namespaces | list | `[]` | Only capture logs from pods in these namespaces (`[]` means all namespaces). |
 
 ### General settings
@@ -55,11 +65,3 @@ Actual integration testing in a live environment should be done in the main [k8s
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | global.platform | string | `""` | The specific platform for this cluster. Will enable compatibility for some platforms. Supported options: (empty) or "openshift". |
-
-### Other Values
-
-| Key | Type | Default | Description |
-|-----|------|---------|-------------|
-| extraDiscoveryRules | string | `""` | Rules to filter pods for log gathering. Only used for "volumes" or "kubernetesApi" gather methods. |
-| extraLogProcessingStages | string | `""` | Stage blocks to be added to the loki.process component for pod logs. ([docs](https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#blocks)) This value is templated so that you can refer to other values from this file. |
-| gatherMethod | string | `"volumes"` | The method to gather pod logs. Options are "volumes", "kubernetesApi", "OpenShiftClusterLogForwarder" (experimental). |

charts/feature-pod-logs/templates/_common_pod_discovery.alloy.tpl

@@ -30,6 +30,20 @@ discovery.relabel "filtered_pods" {
     replacement = "$1"
     target_label = "job"
   }
+{{- range $label, $podLabel := .Values.labels }}
+  rule {
+    source_labels = ["{{ include "pod_label" $podLabel }}"]
+    regex         = "(.+)"
+    target_label  = {{ $label | quote }}
+  }
+{{- end }}
+{{- range $label, $podAnnotation := .Values.annotations }}
+  rule {
+    source_labels = ["{{ include "pod_annotation" $podAnnotation }}"]
+    regex         = "(.+)"
+    target_label  = {{ $label | quote }}
+  }
+{{- end }}
 
   // set the container runtime as a label
   rule {

charts/feature-pod-logs/templates/_helpers.tpl

@@ -15,3 +15,16 @@ If release name contains chart name it will be used as a full name.
 {{- end }}
 {{- end }}
 {{- end }}
+
+{{- define "escape_label" -}}
+{{ . | replace "-" "_" | replace "." "_" | replace "/" "_" }}
+{{- end }}
+
+{{- define "pod_label" -}}
+{{ printf "__meta_kubernetes_pod_label_%s" (include "escape_label" .) }}
+{{- end }}
+
+{{- define "pod_annotation" -}}
+{{ printf "__meta_kubernetes_pod_annotation_%s" (include "escape_label" .) }}
+{{- end }}
+

charts/feature-pod-logs/tests/default_test.yaml

@@ -41,7 +41,17 @@ tests:
                   replacement = "$1"
                   target_label = "job"
                 }
-          
+                rule {
+                  source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
+                  regex         = "(.+)"
+                  target_label  = "app_kubernetes_io_name"
+                }
+                rule {
+                  source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_logs_job"]
+                  regex         = "(.+)"
+                  target_label  = "job"
+                }
+
                 // set the container runtime as a label
                 rule {
                   action = "replace"

charts/feature-pod-logs/values.schema.json

@@ -2,6 +2,14 @@
     "$schema": "http://json-schema.org/schema#",
     "type": "object",
     "properties": {
+        "annotations": {
+            "type": "object",
+            "properties": {
+                "job": {
+                    "type": "string"
+                }
+            }
+        },
         "deployAsConfigMap": {
             "type": "boolean"
         },
@@ -32,6 +40,14 @@
                 }
             }
         },
+        "labels": {
+            "type": "object",
+            "properties": {
+                "app_kubernetes_io_name": {
+                    "type": "string"
+                }
+            }
+        },
         "nameOverride": {
             "type": "string"
         },

charts/feature-pod-logs/values.yaml

@@ -13,22 +13,37 @@ global:
   platform: ""
 
 # -- The method to gather pod logs. Options are "volumes", "kubernetesApi", "OpenShiftClusterLogForwarder" (experimental).
+# @section -- Pod Discovery
 gatherMethod: volumes
 
 # -- Only capture logs from pods in these namespaces (`[]` means all namespaces).
-# @section -- Logs Scrape: Pod Logs
+# @section -- Pod Discovery
 namespaces: []
 
 # -- Do not capture logs from any pods in these namespaces.
-# @section -- Logs Scrape: Pod Logs
+# @section -- Pod Discovery
 excludeNamespaces: []
 
 # -- Rules to filter pods for log gathering. Only used for "volumes" or "kubernetesApi" gather methods.
+# @section -- Pod Discovery
 extraDiscoveryRules: ""
 
+# -- Log labels to set with values copied from the Kubernetes Pod labels.
+# Format: `<log_label>: <kubernetes_label>`.
+# @section -- Log Processing
+labels:
+  app_kubernetes_io_name: app.kubernetes.io/name
+
+# -- Log labels to set with values copied from the Kubernetes Pod annotations.
+# Format: `<log_label>: <kubernetes_annotation>`.
+# @section -- Log Processing
+annotations:
+  job: k8s.grafana.com/logs.job
+
 # -- Stage blocks to be added to the loki.process component for pod logs.
 # ([docs](https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#blocks))
 # This value is templated so that you can refer to other values from this file.
+# @section -- Log Processing
 extraLogProcessingStages: ""
 
 # @ignore

charts/k8s-monitoring/Chart.lock

@@ -42,4 +42,4 @@ dependencies:
   repository: https://grafana.github.io/helm-charts
   version: 0.10.0
 digest: sha256:c42e09be38582ced6f973de7fd7d2f5d96c1e926e187936e0ee1d7ae295c9e0f
-generated: "2024-11-19T10:59:04.581346-06:00"
+generated: "2024-11-21T18:59:30.945426-06:00"