diff --git a/charts/k8s-monitoring/tests/platform/otlp-gateway/.rendered/output.yaml b/charts/k8s-monitoring/tests/platform/otlp-gateway/.rendered/output.yaml index 00ae019637..83d68d6869 100644 --- a/charts/k8s-monitoring/tests/platform/otlp-gateway/.rendered/output.yaml +++ b/charts/k8s-monitoring/tests/platform/otlp-gateway/.rendered/output.yaml @@ -1,4 +1,20 @@ --- +# Source: k8s-monitoring/charts/alloy-logs/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: k8smon-alloy-logs + namespace: default + labels: + helm.sh/chart: alloy-logs-0.11.0 + app.kubernetes.io/name: alloy-logs + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.6.1" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy + app.kubernetes.io/component: rbac +--- # Source: k8s-monitoring/charts/alloy-metrics/templates/serviceaccount.yaml apiVersion: v1 kind: ServiceAccount @@ -59,6 +75,12 @@ data: otelcol.processor.transform "otlp_gateway" { error_mode = "ignore" + log_statements { + context = "log" + statements = [ + "set(resource.attributes[\"service.name\"], attributes[\"container\"]) where resource.attributes[\"service.name\"] == nil", + ] + } output { metrics = [otelcol.processor.batch.otlp_gateway.input] @@ -447,8 +469,343 @@ data: grafana_kubernetes_monitoring_build_info{version="2.0.5", namespace="default"} 1 # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{feature="podLogs", method="volumes", version="1.0.0"} 1 grafana_kubernetes_monitoring_feature_info{feature="integrations", sources="alloy", version="1.0.0"} 1 --- +# Source: k8s-monitoring/templates/alloy-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: k8smon-alloy-logs + namespace: default +data: + config.alloy: |- + // Destination: otlp-gateway (otlp) + otelcol.receiver.prometheus "otlp_gateway" { + output { + metrics = [otelcol.processor.attributes.otlp_gateway.input] + } + } + otelcol.receiver.loki "otlp_gateway" { + output { + logs = [otelcol.processor.attributes.otlp_gateway.input] + } + } + otelcol.auth.basic "otlp_gateway" { + username = nonsensitive(remote.kubernetes.secret.otlp_gateway.data["OTLP_GATEWAY_USER"]) + password = remote.kubernetes.secret.otlp_gateway.data["OTLP_GATEWAY_PASS"] + } + + otelcol.processor.attributes "otlp_gateway" { + action { + key = "cluster" + action = "upsert" + value = "otlp-gateway-test" + } + action { + key = "k8s.cluster.name" + action = "upsert" + value = "otlp-gateway-test" + } + output { + metrics = [otelcol.processor.transform.otlp_gateway.input] + logs = [otelcol.processor.transform.otlp_gateway.input] + traces = [otelcol.processor.transform.otlp_gateway.input] + } + } + + otelcol.processor.transform "otlp_gateway" { + error_mode = "ignore" + log_statements { + context = "log" + statements = [ + "set(resource.attributes[\"service.name\"], attributes[\"container\"]) where resource.attributes[\"service.name\"] == nil", + ] + } + + output { + metrics = [otelcol.processor.batch.otlp_gateway.input] + logs = [otelcol.processor.batch.otlp_gateway.input] + traces = [otelcol.processor.batch.otlp_gateway.input] + } + } + + otelcol.processor.batch "otlp_gateway" { + timeout = "2s" + send_batch_size = 8192 + send_batch_max_size = 0 + + output { + metrics = [otelcol.exporter.otlphttp.otlp_gateway.input] + logs = [otelcol.exporter.otlphttp.otlp_gateway.input] + traces = [otelcol.exporter.otlphttp.otlp_gateway.input] + } + } + otelcol.exporter.otlphttp "otlp_gateway" { + client { + endpoint = "https://otlp-gateway-prod-us-east-0.grafana.net/otlp" + auth = otelcol.auth.basic.otlp_gateway.handler + headers = { + "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.otlp_gateway.data["tenantId"]), + } + tls { + insecure = false + insecure_skip_verify = false + ca_pem = nonsensitive(remote.kubernetes.secret.otlp_gateway.data["ca"]) + cert_pem = nonsensitive(remote.kubernetes.secret.otlp_gateway.data["cert"]) + key_pem = remote.kubernetes.secret.otlp_gateway.data["key"] + } + } + } + + remote.kubernetes.secret "otlp_gateway" { + name = "grafana-cloud-credentials" + namespace = "default" + } + + // Feature: Pod Logs + declare "pod_logs" { + argument "logs_destinations" { + comment = "Must be a list of log destinations where collected logs should be forwarded to" + } + + discovery.relabel "filtered_pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "$1" + target_label = "job" + } + + // set the container runtime as a label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_id"] + regex = "^(\\S+):\\/\\/.+$" + replacement = "$1" + target_label = "tmp_container_runtime" + } + + // set the job label from the k8s.grafana.com/logs.job annotation if it exists + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_logs_job"] + regex = "(.+)" + target_label = "job" + } + + // make all labels on the pod available to the pipeline as labels, + // they are omitted before write to loki via stage.label_keep unless explicitly set + rule { + action = "labelmap" + regex = "__meta_kubernetes_pod_label_(.+)" + } + + // make all annotations on the pod available to the pipeline as labels, + // they are omitted before write to loki via stage.label_keep unless explicitly set + rule { + action = "labelmap" + regex = "__meta_kubernetes_pod_annotation_(.+)" + } + } + + discovery.kubernetes "pods" { + role = "pod" + selectors { + role = "pod" + field = "spec.nodeName=" + sys.env("HOSTNAME") + } + } + + discovery.relabel "filtered_pods_with_paths" { + targets = discovery.relabel.filtered_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "/var/log/pods/*$1/*.log" + target_label = "__path__" + } + } + + local.file_match "pod_logs" { + path_targets = discovery.relabel.filtered_pods_with_paths.output + } + + loki.source.file "pod_logs" { + targets = local.file_match.pod_logs.targets + forward_to = [loki.process.pod_logs.receiver] + } + + loki.process "pod_logs" { + stage.match { + selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" + // the cri processing stage extracts the following k/v pairs: log, stream, time, flags + stage.cri {} + + // Set the extract flags and stream values as labels + stage.labels { + values = { + flags = "", + stream = "", + } + } + } + + stage.match { + selector = "{tmp_container_runtime=\"docker\"}" + // the docker processing stage extracts the following k/v pairs: log, stream, time + stage.docker {} + + // Set the extract stream value as a label + stage.labels { + values = { + stream = "", + } + } + } + + // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, + // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary + // container runtime label as it is no longer needed. + stage.label_drop { + values = [ + "filename", + "tmp_container_runtime", + ] + } + + // Only keep the labels that are defined in the `keepLabels` list. + stage.label_keep { + values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name","integration"] + } + + forward_to = argument.logs_destinations.value + } + } + pod_logs "feature" { + logs_destinations = [ + otelcol.receiver.loki.otlp_gateway.receiver, + ] + } +--- +# Source: k8s-monitoring/charts/alloy-logs/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: k8smon-alloy-logs + labels: + helm.sh/chart: alloy-logs-0.11.0 + app.kubernetes.io/name: alloy-logs + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.6.1" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy + app.kubernetes.io/component: rbac +rules: + # Rules which allow discovery.kubernetes to function. + - apiGroups: + - "" + - "discovery.k8s.io" + - "networking.k8s.io" + resources: + - endpoints + - endpointslices + - ingresses + - nodes + - nodes/proxy + - nodes/metrics + - pods + - services + verbs: + - get + - list + - watch + # Rules which allow loki.source.kubernetes and loki.source.podlogs to work. + - apiGroups: + - "" + resources: + - pods + - pods/log + - namespaces + verbs: + - get + - list + - watch + - apiGroups: + - "monitoring.grafana.com" + resources: + - podlogs + verbs: + - get + - list + - watch + # Rules which allow mimir.rules.kubernetes to work. + - apiGroups: ["monitoring.coreos.com"] + resources: + - prometheusrules + verbs: + - get + - list + - watch + - nonResourceURLs: + - /metrics + verbs: + - get + # Rules for prometheus.kubernetes.* + - apiGroups: ["monitoring.coreos.com"] + resources: + - podmonitors + - servicemonitors + - probes + verbs: + - get + - list + - watch + # Rules which allow eventhandler to work. + - apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + # needed for remote.kubernetes.* + - apiGroups: [""] + resources: + - "configmaps" + - "secrets" + verbs: + - get + - list + - watch + # needed for otelcol.processor.k8sattributes + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["extensions"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] +--- # Source: k8s-monitoring/charts/alloy-metrics/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -549,6 +906,29 @@ rules: resources: ["replicasets"] verbs: ["get", "list", "watch"] --- +# Source: k8s-monitoring/charts/alloy-logs/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: k8smon-alloy-logs + labels: + helm.sh/chart: alloy-logs-0.11.0 + app.kubernetes.io/name: alloy-logs + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.6.1" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy + app.kubernetes.io/component: rbac +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: k8smon-alloy-logs +subjects: + - kind: ServiceAccount + name: k8smon-alloy-logs + namespace: default +--- # Source: k8s-monitoring/charts/alloy-metrics/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -572,6 +952,32 @@ subjects: name: k8smon-alloy-metrics namespace: default --- +# Source: k8s-monitoring/charts/alloy-logs/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: k8smon-alloy-logs + labels: + helm.sh/chart: alloy-logs-0.11.0 + app.kubernetes.io/name: alloy-logs + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.6.1" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy + app.kubernetes.io/component: networking +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: alloy-logs + app.kubernetes.io/instance: k8smon + internalTrafficPolicy: Cluster + ports: + - name: http-metrics + port: 12345 + targetPort: 12345 + protocol: "TCP" +--- # Source: k8s-monitoring/charts/alloy-metrics/templates/cluster_service.yaml apiVersion: v1 kind: Service @@ -631,6 +1037,120 @@ spec: targetPort: 12345 protocol: "TCP" --- +# Source: k8s-monitoring/charts/alloy-logs/templates/controllers/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: k8smon-alloy-logs + labels: + helm.sh/chart: alloy-logs-0.11.0 + app.kubernetes.io/name: alloy-logs + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.6.1" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy +spec: + minReadySeconds: 10 + selector: + matchLabels: + app.kubernetes.io/name: alloy-logs + app.kubernetes.io/instance: k8smon + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: alloy + k8s.grafana.com/logs.job: integrations/alloy + labels: + app.kubernetes.io/name: alloy-logs + app.kubernetes.io/instance: k8smon + spec: + serviceAccountName: k8smon-alloy-logs + containers: + - name: alloy + image: docker.io/grafana/alloy:v1.6.1 + imagePullPolicy: IfNotPresent + args: + - run + - /etc/alloy/config.alloy + - --storage.path=/tmp/alloy + - --server.http.listen-addr=0.0.0.0:12345 + - --server.http.ui-path-prefix=/ + - --stability.level=generally-available + env: + - name: ALLOY_DEPLOY_MODE + value: "helm" + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - containerPort: 12345 + name: http-metrics + readinessProbe: + httpGet: + path: /-/ready + port: 12345 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 1 + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - CHOWN + - DAC_OVERRIDE + - FOWNER + - FSETID + - KILL + - SETGID + - SETUID + - SETPCAP + - NET_BIND_SERVICE + - NET_RAW + - SYS_CHROOT + - MKNOD + - AUDIT_WRITE + - SETFCAP + drop: + - ALL + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: config + mountPath: /etc/alloy + - name: varlog + mountPath: /var/log + readOnly: true + - name: dockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: config-reloader + image: ghcr.io/jimmidyson/configmap-reload:v0.14.0 + args: + - --volume-dir=/etc/alloy + - --webhook-url=http://localhost:12345/-/reload + volumeMounts: + - name: config + mountPath: /etc/alloy + resources: + requests: + cpu: 1m + memory: 5Mi + dnsPolicy: ClusterFirst + nodeSelector: + kubernetes.io/os: linux + volumes: + - name: config + configMap: + name: k8smon-alloy-logs + - name: varlog + hostPath: + path: /var/log + - name: dockercontainers + hostPath: + path: /var/lib/docker/containers +--- # Source: k8s-monitoring/charts/alloy-metrics/templates/controllers/statefulset.yaml apiVersion: apps/v1 kind: StatefulSet