diff --git a/charts/feature-annotation-autodiscovery/templates/_notes.tpl b/charts/feature-annotation-autodiscovery/templates/_notes.tpl index 6aa7d11ecc..94939a73b4 100644 --- a/charts/feature-annotation-autodiscovery/templates/_notes.tpl +++ b/charts/feature-annotation-autodiscovery/templates/_notes.tpl @@ -5,3 +5,7 @@ Scrape metrics from pods and services with the "{{.Values.annotations.scrape}}: {{- end }} {{- define "feature.annotationAutodiscovery.notes.actions" }}{{- end }} + +{{- define "feature.annotationAutodiscovery.summary" -}} +version: {{ .Chart.Version }} +{{- end }} diff --git a/charts/feature-application-observability/templates/_notes.tpl b/charts/feature-application-observability/templates/_notes.tpl index 6140c4cba5..e3f754c682 100644 --- a/charts/feature-application-observability/templates/_notes.tpl +++ b/charts/feature-application-observability/templates/_notes.tpl @@ -21,3 +21,12 @@ Configure your applications to send telemetry data to: * http://{{ .Collector.ServiceName }}.{{ .Collector.Namespace }}.svc.cluster.local:{{ .Values.receivers.zipkin.port }} (Zipkin) {{ end }} {{- end }} + +{{- define "feature.applicationObservability.summary" -}} +{{- $receivers := list }} +{{- if .Values.receivers.grpc.enabled }}{{- $receivers = append $receivers "otlpgrpc" }}{{ end }} +{{- if .Values.receivers.http.enabled }}{{- $receivers = append $receivers "otlphttp" }}{{ end }} +{{- if .Values.receivers.zipkin.enabled }}{{- $receivers = append $receivers "zipkin" }}{{ end }} +version: {{ .Chart.Version }} +protocols: {{ $receivers | join "," }} +{{- end }} diff --git a/charts/feature-cluster-events/templates/_notes.tpl b/charts/feature-cluster-events/templates/_notes.tpl index 6447c8e4d1..b5aa085606 100644 --- a/charts/feature-cluster-events/templates/_notes.tpl +++ b/charts/feature-cluster-events/templates/_notes.tpl @@ -5,3 +5,7 @@ Gather Kubernetes Cluster events{{- if .Values.namespaces }} from the namespaces {{- end }} {{- define "feature.clusterEvents.notes.actions" }}{{- end }} + +{{- define "feature.clusterEvents.summary" -}} +version: {{ .Chart.Version }} +{{- end }} diff --git a/charts/feature-cluster-metrics/templates/_notes.tpl b/charts/feature-cluster-metrics/templates/_notes.tpl index d9dd4f3264..b592235be2 100644 --- a/charts/feature-cluster-metrics/templates/_notes.tpl +++ b/charts/feature-cluster-metrics/templates/_notes.tpl @@ -18,3 +18,28 @@ Scrape Kubernetes Cluster metrics {{- end }} {{- define "feature.clusterMetrics.notes.actions" }}{{- end }} + +{{- define "feature.clusterMetrics.summary" -}} +{{- $sources := list }} +{{- if .Values.controlPlane.enabled }}{{- $sources = append $sources "controlPlane" }}{{ end }} +{{- if .Values.kubelet.enabled }}{{- $sources = append $sources "kubelet" }}{{ end }} +{{- if .Values.kubeletResource.enabled }}{{- $sources = append $sources "kubeletResource" }}{{ end }} +{{- if .Values.cadvisor.enabled }}{{- $sources = append $sources "cadvisor" }}{{ end }} +{{- if .Values.apiServer.enabled }}{{- $sources = append $sources "apiServer" }}{{ end }} +{{- if .Values.kubeControllerManager.enabled }}{{- $sources = append $sources "kubeControllerManager" }}{{ end }} +{{- if .Values.kubeProxy.enabled }}{{- $sources = append $sources "kubeProxy" }}{{ end }} +{{- if .Values.kubeScheduler.enabled }}{{- $sources = append $sources "kubeScheduler" }}{{ end }} +{{- if (index .Values "kube-state-metrics").enabled }}{{- $sources = append $sources "kube-state-metrics" }}{{ end }} +{{- if (index .Values "node-exporter").enabled }}{{- $sources = append $sources "node-exporter" }}{{ end }} +{{- if (index .Values "windows-exporter").enabled }}{{- $sources = append $sources "windows-exporter" }}{{ end }} +{{- if .Values.kepler.enabled }}{{- $sources = append $sources "kepler" }}{{ end }} + +{{- $deployments := list }} +{{- if (index .Values "kube-state-metrics").deploy }}{{- $deployments = append $deployments "kube-state-metrics" }}{{ end }} +{{- if (index .Values "node-exporter").deploy }}{{- $deployments = append $deployments "node-exporter" }}{{ end }} +{{- if (index .Values "windows-exporter").deploy }}{{- $deployments = append $deployments "windows-exporter" }}{{ end }} +{{- if .Values.kepler.enabled }}{{- $deployments = append $deployments "kepler" }}{{ end }} +version: {{ .Chart.Version }} +sources: {{ $sources | join "," }} +deployments: {{ $deployments | join "," }} +{{- end }} diff --git a/charts/feature-frontend-observability/templates/_notes.tpl b/charts/feature-frontend-observability/templates/_notes.tpl index 2a752a9f5f..32c7a7c37a 100644 --- a/charts/feature-frontend-observability/templates/_notes.tpl +++ b/charts/feature-frontend-observability/templates/_notes.tpl @@ -5,3 +5,7 @@ Receive data from Frontend applications {{- end }} {{- define "feature.frontendObservability.notes.actions" }}{{- end }} + +{{- define "feature.frontendObservability.summary" -}} +version: {{ .Chart.Version }} +{{- end }} diff --git a/charts/feature-integrations/templates/_notes.tpl b/charts/feature-integrations/templates/_notes.tpl index 3042cd3cbc..134e8f2a22 100644 --- a/charts/feature-integrations/templates/_notes.tpl +++ b/charts/feature-integrations/templates/_notes.tpl @@ -6,3 +6,7 @@ Gather data from service integrations {{- end }} {{- define "feature.integrations.notes.actions" }}{{- end }} + +{{- define "feature.integrations.summary" -}} +version: {{ .Chart.Version }} +{{- end }} diff --git a/charts/feature-pod-logs/templates/_notes.tpl b/charts/feature-pod-logs/templates/_notes.tpl index 4b9bc0186f..67b320794a 100644 --- a/charts/feature-pod-logs/templates/_notes.tpl +++ b/charts/feature-pod-logs/templates/_notes.tpl @@ -6,3 +6,7 @@ Gather logs from Kubernetes Pods {{- define "feature.podLogs.notes.actions" }}{{- end }} +{{- define "feature.podLogs.summary" -}} +version: {{ .Chart.Version }} +method: {{ .Values.gatherMethod }} +{{- end }} diff --git a/charts/feature-profiling/templates/_notes.tpl b/charts/feature-profiling/templates/_notes.tpl index 1c9bb0c969..c85c46efff 100644 --- a/charts/feature-profiling/templates/_notes.tpl +++ b/charts/feature-profiling/templates/_notes.tpl @@ -5,3 +5,8 @@ Gather profiles {{- end }} {{- define "feature.profiling.notes.actions" }}{{- end }} + +{{- define "feature.profiling.summary" -}} +version: {{ .Chart.Version }} +method: {{ .Values.gatherMethod }} +{{- end }} diff --git a/charts/feature-prometheus-operator-objects/templates/_notes.tpl b/charts/feature-prometheus-operator-objects/templates/_notes.tpl index 9ab81ebcf5..46b83e91b7 100644 --- a/charts/feature-prometheus-operator-objects/templates/_notes.tpl +++ b/charts/feature-prometheus-operator-objects/templates/_notes.tpl @@ -13,3 +13,7 @@ Scrapes metrics from {{ include "english_list" $sources }}. {{- end }} {{- define "feature.prometheusOperatorObjects.notes.actions" }}{{- end }} + +{{- define "feature.prometheusOperatorObjects.summary" -}} +version: {{ .Chart.Version }} +{{- end }} diff --git a/charts/k8s-monitoring/Chart.lock b/charts/k8s-monitoring/Chart.lock index 911a7907a3..03af10372b 100644 --- a/charts/k8s-monitoring/Chart.lock +++ b/charts/k8s-monitoring/Chart.lock @@ -42,4 +42,4 @@ dependencies: repository: https://grafana.github.io/helm-charts version: 0.9.1 digest: sha256:734d4c8f6076481eb580378daa65fe163c78e9e07a1a214cb4b2fed16441b4c9 -generated: "2024-10-16T09:29:39.056825-05:00" +generated: "2024-10-21T17:06:40.407816+02:00" diff --git a/charts/k8s-monitoring/README.md b/charts/k8s-monitoring/README.md index accd13d622..d7779b2e94 100644 --- a/charts/k8s-monitoring/README.md +++ b/charts/k8s-monitoring/README.md @@ -291,6 +291,14 @@ podLogs: | prometheusOperatorObjects.destinations | list | `[]` | The destinations where metrics will be sent. If empty, all metrics-capable destinations will be used. | | prometheusOperatorObjects.enabled | bool | `false` | Enable gathering metrics from Prometheus Operator Objects. | +### Features - Self-reporting + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| selfReporting | object | `{"enabled":true,"scrapeInterval":"1h"}` | Self-reporting creates a single metric and log that reports anonymized information about how this Helm chart was configured. It reports features enabled, destinations types used, and alloy instances enabled. It does not report any actual telemetry data, credentials or configuration, or send any data to any destination other than the ones configured above. | +| selfReporting.enabled | bool | `true` | Enable Self-reporting. | +| selfReporting.scrapeInterval | string | `"1h"` | How frequently to generate self-report metrics. This does utilize the global scrapeInterval setting. | + ### Other Values | Key | Type | Default | Description | diff --git a/charts/k8s-monitoring/charts/k8s-monitoring-feature-annotation-autodiscovery-1.0.0.tgz b/charts/k8s-monitoring/charts/k8s-monitoring-feature-annotation-autodiscovery-1.0.0.tgz index 0d30062719..2b9117578b 100644 Binary files a/charts/k8s-monitoring/charts/k8s-monitoring-feature-annotation-autodiscovery-1.0.0.tgz and b/charts/k8s-monitoring/charts/k8s-monitoring-feature-annotation-autodiscovery-1.0.0.tgz differ diff --git a/charts/k8s-monitoring/charts/k8s-monitoring-feature-application-observability-1.0.0.tgz b/charts/k8s-monitoring/charts/k8s-monitoring-feature-application-observability-1.0.0.tgz index cb224b97f1..bf43e4ccf4 100644 Binary files a/charts/k8s-monitoring/charts/k8s-monitoring-feature-application-observability-1.0.0.tgz and b/charts/k8s-monitoring/charts/k8s-monitoring-feature-application-observability-1.0.0.tgz differ diff --git a/charts/k8s-monitoring/charts/k8s-monitoring-feature-cluster-events-1.0.0.tgz b/charts/k8s-monitoring/charts/k8s-monitoring-feature-cluster-events-1.0.0.tgz index f93ae02b9d..7589cbff90 100644 Binary files a/charts/k8s-monitoring/charts/k8s-monitoring-feature-cluster-events-1.0.0.tgz and b/charts/k8s-monitoring/charts/k8s-monitoring-feature-cluster-events-1.0.0.tgz differ diff --git a/charts/k8s-monitoring/charts/k8s-monitoring-feature-cluster-metrics-1.0.0.tgz b/charts/k8s-monitoring/charts/k8s-monitoring-feature-cluster-metrics-1.0.0.tgz index 7cc148bc18..49b6fc49a6 100644 Binary files a/charts/k8s-monitoring/charts/k8s-monitoring-feature-cluster-metrics-1.0.0.tgz and b/charts/k8s-monitoring/charts/k8s-monitoring-feature-cluster-metrics-1.0.0.tgz differ diff --git a/charts/k8s-monitoring/charts/k8s-monitoring-feature-frontend-observability-1.0.0.tgz b/charts/k8s-monitoring/charts/k8s-monitoring-feature-frontend-observability-1.0.0.tgz index 552111e694..b2c7fd8b49 100644 Binary files a/charts/k8s-monitoring/charts/k8s-monitoring-feature-frontend-observability-1.0.0.tgz and b/charts/k8s-monitoring/charts/k8s-monitoring-feature-frontend-observability-1.0.0.tgz differ diff --git a/charts/k8s-monitoring/charts/k8s-monitoring-feature-integrations-1.0.0.tgz b/charts/k8s-monitoring/charts/k8s-monitoring-feature-integrations-1.0.0.tgz index 22b3240397..f93add1d02 100644 Binary files a/charts/k8s-monitoring/charts/k8s-monitoring-feature-integrations-1.0.0.tgz and b/charts/k8s-monitoring/charts/k8s-monitoring-feature-integrations-1.0.0.tgz differ diff --git a/charts/k8s-monitoring/charts/k8s-monitoring-feature-pod-logs-1.0.0.tgz b/charts/k8s-monitoring/charts/k8s-monitoring-feature-pod-logs-1.0.0.tgz index f06398080c..6f379b9a68 100644 Binary files a/charts/k8s-monitoring/charts/k8s-monitoring-feature-pod-logs-1.0.0.tgz and b/charts/k8s-monitoring/charts/k8s-monitoring-feature-pod-logs-1.0.0.tgz differ diff --git a/charts/k8s-monitoring/charts/k8s-monitoring-feature-profiling-1.0.0.tgz b/charts/k8s-monitoring/charts/k8s-monitoring-feature-profiling-1.0.0.tgz index f0d2f0c319..143ae215f3 100644 Binary files a/charts/k8s-monitoring/charts/k8s-monitoring-feature-profiling-1.0.0.tgz and b/charts/k8s-monitoring/charts/k8s-monitoring-feature-profiling-1.0.0.tgz differ diff --git a/charts/k8s-monitoring/charts/k8s-monitoring-feature-prometheus-operator-objects-1.0.0.tgz b/charts/k8s-monitoring/charts/k8s-monitoring-feature-prometheus-operator-objects-1.0.0.tgz index 1d919b0c67..3551f27854 100644 Binary files a/charts/k8s-monitoring/charts/k8s-monitoring-feature-prometheus-operator-objects-1.0.0.tgz and b/charts/k8s-monitoring/charts/k8s-monitoring-feature-prometheus-operator-objects-1.0.0.tgz differ diff --git a/charts/k8s-monitoring/docs/examples/auth/bearer-token/alloy-logs.alloy b/charts/k8s-monitoring/docs/examples/auth/bearer-token/alloy-logs.alloy index 7e70387d52..4e020ee5d4 100644 --- a/charts/k8s-monitoring/docs/examples/auth/bearer-token/alloy-logs.alloy +++ b/charts/k8s-monitoring/docs/examples/auth/bearer-token/alloy-logs.alloy @@ -13,6 +13,7 @@ loki.write "loki" { "k8s_cluster_name" = "bearer-token-example-cluster", } } + // Feature: Pod Logs declare "pod_logs" { argument "logs_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/auth/bearer-token/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/auth/bearer-token/alloy-metrics.alloy index 27e1afde21..60cc56dba7 100644 --- a/charts/k8s-monitoring/docs/examples/auth/bearer-token/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/auth/bearer-token/alloy-metrics.alloy @@ -48,7 +48,6 @@ remote.kubernetes.secret "prometheus" { namespace = "default" } - // Feature: Prometheus Operator Objects declare "prometheus_operator_objects" { argument "metrics_destinations" { @@ -93,3 +92,37 @@ prometheus_operator_objects "feature" { prometheus.remote_write.prometheus.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/auth/bearer-token/alloy-receiver.alloy b/charts/k8s-monitoring/docs/examples/auth/bearer-token/alloy-receiver.alloy index 9de0f57ed7..317b1c0ca3 100644 --- a/charts/k8s-monitoring/docs/examples/auth/bearer-token/alloy-receiver.alloy +++ b/charts/k8s-monitoring/docs/examples/auth/bearer-token/alloy-receiver.alloy @@ -110,7 +110,6 @@ remote.kubernetes.secret "tempo" { namespace = "tempo" } - // Feature: Application Observability declare "application_observability" { argument "metrics_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/auth/bearer-token/output.yaml b/charts/k8s-monitoring/docs/examples/auth/bearer-token/output.yaml index 85ef63df89..94acb83543 100644 --- a/charts/k8s-monitoring/docs/examples/auth/bearer-token/output.yaml +++ b/charts/k8s-monitoring/docs/examples/auth/bearer-token/output.yaml @@ -57,7 +57,155 @@ type: Opaque data: bearerToken: "c2FtcGxlLWJlYXJlci10b2tlbg==" --- -# Source: k8s-monitoring/templates/alloy-logs-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: ko-alloy-metrics + namespace: default +data: + config.alloy: |- + // Destination: prometheus (prometheus) + otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] + } + + prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.prometheus.data["tenantId"]), + } + bearer_token = remote.kubernetes.secret.prometheus.data["bearerToken"] + tls_config { + insecure_skip_verify = false + ca_pem = nonsensitive(remote.kubernetes.secret.prometheus.data["ca"]) + cert_pem = nonsensitive(remote.kubernetes.secret.prometheus.data["cert"]) + key_pem = remote.kubernetes.secret.prometheus.data["key"] + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "bearer-token-example-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "bearer-token-example-cluster" + target_label = "cluster" + } + } + } + + remote.kubernetes.secret "prometheus" { + name = "prometheus-ko-k8s-monitoring" + namespace = "default" + } + + // Feature: Prometheus Operator Objects + declare "prometheus_operator_objects" { + argument "metrics_destinations" { + comment = "Must be a list of metric destinations where collected metrics should be forwarded to" + } + + // Prometheus Operator podMonitor objects + prometheus.operator.podmonitors "pod_monitors" { + clustering { + enabled = true + } + scrape { + default_scrape_interval = "60s" + } + forward_to = argument.metrics_destinations.value + } + + // Prometheus Operator podMonitor objects + prometheus.operator.probes "pod_monitors" { + clustering { + enabled = true + } + scrape { + default_scrape_interval = "60s" + } + forward_to = argument.metrics_destinations.value + } + + // Prometheus Operator ServiceMonitor objects + prometheus.operator.servicemonitors "service_monitors" { + clustering { + enabled = true + } + scrape { + default_scrape_interval = "60s" + } + forward_to = argument.metrics_destinations.value + } + } + prometheus_operator_objects "feature" { + metrics_destinations = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{feature="applicationObservability", protocols="otlpgrpc", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="podLogs", method="volumes", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="prometheusOperatorObjects", version="1.0.0"} 1 +--- +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -80,6 +228,7 @@ data: "k8s_cluster_name" = "bearer-token-example-cluster", } } + // Feature: Pod Logs declare "pod_logs" { argument "logs_destinations" { @@ -196,111 +345,7 @@ data: ] } --- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: ko-alloy-metrics - namespace: default -data: - config.alloy: |- - // Destination: prometheus (prometheus) - otelcol.exporter.prometheus "prometheus" { - forward_to = [prometheus.remote_write.prometheus.receiver] - } - - prometheus.remote_write "prometheus" { - endpoint { - url = "http://prometheus.prometheus.svc:9090/api/v1/write" - headers = { - "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.prometheus.data["tenantId"]), - } - bearer_token = remote.kubernetes.secret.prometheus.data["bearerToken"] - tls_config { - insecure_skip_verify = false - ca_pem = nonsensitive(remote.kubernetes.secret.prometheus.data["ca"]) - cert_pem = nonsensitive(remote.kubernetes.secret.prometheus.data["cert"]) - key_pem = remote.kubernetes.secret.prometheus.data["key"] - } - send_native_histograms = false - queue_config { - capacity = 10000 - min_shards = 1 - max_shards = 50 - max_samples_per_send = 2000 - batch_send_deadline = "5s" - min_backoff = "30ms" - max_backoff = "5s" - retry_on_http_429 = true - sample_age_limit = "0s" - } - write_relabel_config { - source_labels = ["cluster"] - regex = "" - replacement = "bearer-token-example-cluster" - target_label = "cluster" - } - write_relabel_config { - source_labels = ["k8s.cluster.name"] - regex = "" - replacement = "bearer-token-example-cluster" - target_label = "cluster" - } - } - } - - remote.kubernetes.secret "prometheus" { - name = "prometheus-ko-k8s-monitoring" - namespace = "default" - } - - - // Feature: Prometheus Operator Objects - declare "prometheus_operator_objects" { - argument "metrics_destinations" { - comment = "Must be a list of metric destinations where collected metrics should be forwarded to" - } - - // Prometheus Operator podMonitor objects - prometheus.operator.podmonitors "pod_monitors" { - clustering { - enabled = true - } - scrape { - default_scrape_interval = "60s" - } - forward_to = argument.metrics_destinations.value - } - - // Prometheus Operator podMonitor objects - prometheus.operator.probes "pod_monitors" { - clustering { - enabled = true - } - scrape { - default_scrape_interval = "60s" - } - forward_to = argument.metrics_destinations.value - } - - // Prometheus Operator ServiceMonitor objects - prometheus.operator.servicemonitors "service_monitors" { - clustering { - enabled = true - } - scrape { - default_scrape_interval = "60s" - } - forward_to = argument.metrics_destinations.value - } - } - prometheus_operator_objects "feature" { - metrics_destinations = [ - prometheus.remote_write.prometheus.receiver, - ] - } ---- -# Source: k8s-monitoring/templates/alloy-receiver-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -420,7 +465,6 @@ data: namespace = "tempo" } - // Feature: Application Observability declare "application_observability" { argument "metrics_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/auth/embedded-secrets/alloy-logs.alloy b/charts/k8s-monitoring/docs/examples/auth/embedded-secrets/alloy-logs.alloy index da37bd8038..7b575b9944 100644 --- a/charts/k8s-monitoring/docs/examples/auth/embedded-secrets/alloy-logs.alloy +++ b/charts/k8s-monitoring/docs/examples/auth/embedded-secrets/alloy-logs.alloy @@ -14,6 +14,7 @@ loki.write "loki" { "k8s_cluster_name" = "embedded-secrets-example-cluster", } } + // Feature: Pod Logs declare "pod_logs" { argument "logs_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/auth/embedded-secrets/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/auth/embedded-secrets/alloy-metrics.alloy index 4cf9b4f977..9da3ee16f3 100644 --- a/charts/k8s-monitoring/docs/examples/auth/embedded-secrets/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/auth/embedded-secrets/alloy-metrics.alloy @@ -46,7 +46,6 @@ prometheus.remote_write "prometheus" { } } - // Feature: Prometheus Operator Objects declare "prometheus_operator_objects" { argument "metrics_destinations" { @@ -91,3 +90,37 @@ prometheus_operator_objects "feature" { prometheus.remote_write.prometheus.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/auth/embedded-secrets/alloy-receiver.alloy b/charts/k8s-monitoring/docs/examples/auth/embedded-secrets/alloy-receiver.alloy index f9dba560e2..751f81a697 100644 --- a/charts/k8s-monitoring/docs/examples/auth/embedded-secrets/alloy-receiver.alloy +++ b/charts/k8s-monitoring/docs/examples/auth/embedded-secrets/alloy-receiver.alloy @@ -104,7 +104,6 @@ otelcol.exporter.otlp "tempo" { } } - // Feature: Application Observability declare "application_observability" { argument "metrics_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/auth/embedded-secrets/output.yaml b/charts/k8s-monitoring/docs/examples/auth/embedded-secrets/output.yaml index ba1c6fe720..c13ebdc05a 100644 --- a/charts/k8s-monitoring/docs/examples/auth/embedded-secrets/output.yaml +++ b/charts/k8s-monitoring/docs/examples/auth/embedded-secrets/output.yaml @@ -47,7 +47,153 @@ metadata: app.kubernetes.io/part-of: alloy app.kubernetes.io/component: rbac --- -# Source: k8s-monitoring/templates/alloy-logs-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: ko-alloy-metrics + namespace: default +data: + config.alloy: |- + // Destination: prometheus (prometheus) + otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] + } + + prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + } + sigv4 { + access_key = "my-access-key" + region = "ap-southeast-2" + secret_key = "my-secret-key" + } + tls_config { + insecure_skip_verify = false + ca_pem = "" + cert_pem = "" + key_pem = "" + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "embedded-secrets-example-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "embedded-secrets-example-cluster" + target_label = "cluster" + } + } + } + + // Feature: Prometheus Operator Objects + declare "prometheus_operator_objects" { + argument "metrics_destinations" { + comment = "Must be a list of metric destinations where collected metrics should be forwarded to" + } + + // Prometheus Operator podMonitor objects + prometheus.operator.podmonitors "pod_monitors" { + clustering { + enabled = true + } + scrape { + default_scrape_interval = "60s" + } + forward_to = argument.metrics_destinations.value + } + + // Prometheus Operator podMonitor objects + prometheus.operator.probes "pod_monitors" { + clustering { + enabled = true + } + scrape { + default_scrape_interval = "60s" + } + forward_to = argument.metrics_destinations.value + } + + // Prometheus Operator ServiceMonitor objects + prometheus.operator.servicemonitors "service_monitors" { + clustering { + enabled = true + } + scrape { + default_scrape_interval = "60s" + } + forward_to = argument.metrics_destinations.value + } + } + prometheus_operator_objects "feature" { + metrics_destinations = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{feature="applicationObservability", protocols="otlpgrpc", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="podLogs", method="volumes", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="prometheusOperatorObjects", version="1.0.0"} 1 +--- +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -71,6 +217,7 @@ data: "k8s_cluster_name" = "embedded-secrets-example-cluster", } } + // Feature: Pod Logs declare "pod_logs" { argument "logs_destinations" { @@ -187,109 +334,7 @@ data: ] } --- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: ko-alloy-metrics - namespace: default -data: - config.alloy: |- - // Destination: prometheus (prometheus) - otelcol.exporter.prometheus "prometheus" { - forward_to = [prometheus.remote_write.prometheus.receiver] - } - - prometheus.remote_write "prometheus" { - endpoint { - url = "http://prometheus.prometheus.svc:9090/api/v1/write" - headers = { - } - sigv4 { - access_key = "my-access-key" - region = "ap-southeast-2" - secret_key = "my-secret-key" - } - tls_config { - insecure_skip_verify = false - ca_pem = "" - cert_pem = "" - key_pem = "" - } - send_native_histograms = false - queue_config { - capacity = 10000 - min_shards = 1 - max_shards = 50 - max_samples_per_send = 2000 - batch_send_deadline = "5s" - min_backoff = "30ms" - max_backoff = "5s" - retry_on_http_429 = true - sample_age_limit = "0s" - } - write_relabel_config { - source_labels = ["cluster"] - regex = "" - replacement = "embedded-secrets-example-cluster" - target_label = "cluster" - } - write_relabel_config { - source_labels = ["k8s.cluster.name"] - regex = "" - replacement = "embedded-secrets-example-cluster" - target_label = "cluster" - } - } - } - - - // Feature: Prometheus Operator Objects - declare "prometheus_operator_objects" { - argument "metrics_destinations" { - comment = "Must be a list of metric destinations where collected metrics should be forwarded to" - } - - // Prometheus Operator podMonitor objects - prometheus.operator.podmonitors "pod_monitors" { - clustering { - enabled = true - } - scrape { - default_scrape_interval = "60s" - } - forward_to = argument.metrics_destinations.value - } - - // Prometheus Operator podMonitor objects - prometheus.operator.probes "pod_monitors" { - clustering { - enabled = true - } - scrape { - default_scrape_interval = "60s" - } - forward_to = argument.metrics_destinations.value - } - - // Prometheus Operator ServiceMonitor objects - prometheus.operator.servicemonitors "service_monitors" { - clustering { - enabled = true - } - scrape { - default_scrape_interval = "60s" - } - forward_to = argument.metrics_destinations.value - } - } - prometheus_operator_objects "feature" { - metrics_destinations = [ - prometheus.remote_write.prometheus.receiver, - ] - } ---- -# Source: k8s-monitoring/templates/alloy-receiver-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -403,7 +448,6 @@ data: } } - // Feature: Application Observability declare "application_observability" { argument "metrics_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/auth/external-secrets/alloy-logs.alloy b/charts/k8s-monitoring/docs/examples/auth/external-secrets/alloy-logs.alloy index 8bbd0014c7..afdce92b1c 100644 --- a/charts/k8s-monitoring/docs/examples/auth/external-secrets/alloy-logs.alloy +++ b/charts/k8s-monitoring/docs/examples/auth/external-secrets/alloy-logs.alloy @@ -22,6 +22,7 @@ remote.kubernetes.secret "loki" { name = "my-monitoring-secret" namespace = "monitoring" } + // Feature: Pod Logs declare "pod_logs" { argument "logs_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/auth/external-secrets/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/auth/external-secrets/alloy-metrics.alloy index cb331d423e..22e9a3eab6 100644 --- a/charts/k8s-monitoring/docs/examples/auth/external-secrets/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/auth/external-secrets/alloy-metrics.alloy @@ -51,7 +51,6 @@ remote.kubernetes.secret "prometheus" { namespace = "monitoring" } - // Feature: Prometheus Operator Objects declare "prometheus_operator_objects" { argument "metrics_destinations" { @@ -96,3 +95,37 @@ prometheus_operator_objects "feature" { prometheus.remote_write.prometheus.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/auth/external-secrets/alloy-receiver.alloy b/charts/k8s-monitoring/docs/examples/auth/external-secrets/alloy-receiver.alloy index e1461cb027..12bc04aefe 100644 --- a/charts/k8s-monitoring/docs/examples/auth/external-secrets/alloy-receiver.alloy +++ b/charts/k8s-monitoring/docs/examples/auth/external-secrets/alloy-receiver.alloy @@ -123,7 +123,6 @@ remote.kubernetes.secret "tempo" { namespace = "tempo" } - // Feature: Application Observability declare "application_observability" { argument "metrics_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/auth/external-secrets/output.yaml b/charts/k8s-monitoring/docs/examples/auth/external-secrets/output.yaml index 7bd9e260ac..000bd0ea12 100644 --- a/charts/k8s-monitoring/docs/examples/auth/external-secrets/output.yaml +++ b/charts/k8s-monitoring/docs/examples/auth/external-secrets/output.yaml @@ -47,7 +47,158 @@ metadata: app.kubernetes.io/part-of: alloy app.kubernetes.io/component: rbac --- -# Source: k8s-monitoring/templates/alloy-logs-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: ko-alloy-metrics + namespace: default +data: + config.alloy: |- + // Destination: prometheus (prometheus) + otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] + } + + prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.prometheus.data["tenantId"]), + } + basic_auth { + username = nonsensitive(remote.kubernetes.secret.prometheus.data["prom-username"]) + password = remote.kubernetes.secret.prometheus.data["access-token"] + } + tls_config { + insecure_skip_verify = false + ca_pem = nonsensitive(remote.kubernetes.secret.prometheus.data["ca"]) + cert_pem = nonsensitive(remote.kubernetes.secret.prometheus.data["cert"]) + key_pem = remote.kubernetes.secret.prometheus.data["key"] + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "external-secrets-example-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "external-secrets-example-cluster" + target_label = "cluster" + } + } + } + + remote.kubernetes.secret "prometheus" { + name = "my-monitoring-secret" + namespace = "monitoring" + } + + // Feature: Prometheus Operator Objects + declare "prometheus_operator_objects" { + argument "metrics_destinations" { + comment = "Must be a list of metric destinations where collected metrics should be forwarded to" + } + + // Prometheus Operator podMonitor objects + prometheus.operator.podmonitors "pod_monitors" { + clustering { + enabled = true + } + scrape { + default_scrape_interval = "60s" + } + forward_to = argument.metrics_destinations.value + } + + // Prometheus Operator podMonitor objects + prometheus.operator.probes "pod_monitors" { + clustering { + enabled = true + } + scrape { + default_scrape_interval = "60s" + } + forward_to = argument.metrics_destinations.value + } + + // Prometheus Operator ServiceMonitor objects + prometheus.operator.servicemonitors "service_monitors" { + clustering { + enabled = true + } + scrape { + default_scrape_interval = "60s" + } + forward_to = argument.metrics_destinations.value + } + } + prometheus_operator_objects "feature" { + metrics_destinations = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{feature="applicationObservability", protocols="otlpgrpc", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="podLogs", method="volumes", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="prometheusOperatorObjects", version="1.0.0"} 1 +--- +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -79,6 +230,7 @@ data: name = "my-monitoring-secret" namespace = "monitoring" } + // Feature: Pod Logs declare "pod_logs" { argument "logs_destinations" { @@ -195,114 +347,7 @@ data: ] } --- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: ko-alloy-metrics - namespace: default -data: - config.alloy: |- - // Destination: prometheus (prometheus) - otelcol.exporter.prometheus "prometheus" { - forward_to = [prometheus.remote_write.prometheus.receiver] - } - - prometheus.remote_write "prometheus" { - endpoint { - url = "http://prometheus.prometheus.svc:9090/api/v1/write" - headers = { - "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.prometheus.data["tenantId"]), - } - basic_auth { - username = nonsensitive(remote.kubernetes.secret.prometheus.data["prom-username"]) - password = remote.kubernetes.secret.prometheus.data["access-token"] - } - tls_config { - insecure_skip_verify = false - ca_pem = nonsensitive(remote.kubernetes.secret.prometheus.data["ca"]) - cert_pem = nonsensitive(remote.kubernetes.secret.prometheus.data["cert"]) - key_pem = remote.kubernetes.secret.prometheus.data["key"] - } - send_native_histograms = false - queue_config { - capacity = 10000 - min_shards = 1 - max_shards = 50 - max_samples_per_send = 2000 - batch_send_deadline = "5s" - min_backoff = "30ms" - max_backoff = "5s" - retry_on_http_429 = true - sample_age_limit = "0s" - } - write_relabel_config { - source_labels = ["cluster"] - regex = "" - replacement = "external-secrets-example-cluster" - target_label = "cluster" - } - write_relabel_config { - source_labels = ["k8s.cluster.name"] - regex = "" - replacement = "external-secrets-example-cluster" - target_label = "cluster" - } - } - } - - remote.kubernetes.secret "prometheus" { - name = "my-monitoring-secret" - namespace = "monitoring" - } - - - // Feature: Prometheus Operator Objects - declare "prometheus_operator_objects" { - argument "metrics_destinations" { - comment = "Must be a list of metric destinations where collected metrics should be forwarded to" - } - - // Prometheus Operator podMonitor objects - prometheus.operator.podmonitors "pod_monitors" { - clustering { - enabled = true - } - scrape { - default_scrape_interval = "60s" - } - forward_to = argument.metrics_destinations.value - } - - // Prometheus Operator podMonitor objects - prometheus.operator.probes "pod_monitors" { - clustering { - enabled = true - } - scrape { - default_scrape_interval = "60s" - } - forward_to = argument.metrics_destinations.value - } - - // Prometheus Operator ServiceMonitor objects - prometheus.operator.servicemonitors "service_monitors" { - clustering { - enabled = true - } - scrape { - default_scrape_interval = "60s" - } - forward_to = argument.metrics_destinations.value - } - } - prometheus_operator_objects "feature" { - metrics_destinations = [ - prometheus.remote_write.prometheus.receiver, - ] - } ---- -# Source: k8s-monitoring/templates/alloy-receiver-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -435,7 +480,6 @@ data: namespace = "tempo" } - // Feature: Application Observability declare "application_observability" { argument "metrics_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/auth/sigv4/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/auth/sigv4/alloy-metrics.alloy index f6def1a04b..94a50bd1fc 100644 --- a/charts/k8s-monitoring/docs/examples/auth/sigv4/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/auth/sigv4/alloy-metrics.alloy @@ -50,6 +50,7 @@ remote.kubernetes.secret "prometheus" { name = "prometheus-ko-k8s-monitoring" namespace = "default" } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -253,3 +254,37 @@ cluster_metrics "feature" { prometheus.remote_write.prometheus.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/auth/sigv4/output.yaml b/charts/k8s-monitoring/docs/examples/auth/sigv4/output.yaml index 26d77f482b..c9067af3d4 100644 --- a/charts/k8s-monitoring/docs/examples/auth/sigv4/output.yaml +++ b/charts/k8s-monitoring/docs/examples/auth/sigv4/output.yaml @@ -99,7 +99,7 @@ data: service: services-where: "Name='containerd' or Name='kubelet'" --- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -159,6 +159,7 @@ data: name = "prometheus-ko-k8s-monitoring" namespace = "default" } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -362,6 +363,49 @@ data: prometheus.remote_write.prometheus.receiver, ] } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{deployments="kube-state-metrics,node-exporter,windows-exporter", feature="clusterMetrics", sources="kubelet,kubeletResource,cadvisor,kube-state-metrics,node-exporter,windows-exporter", version="1.0.0"} 1 --- # Source: k8s-monitoring/charts/alloy-metrics/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/autoscaling/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/autoscaling/alloy-metrics.alloy index 98d147e26c..4ff7e34c1f 100644 --- a/charts/k8s-monitoring/docs/examples/autoscaling/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/autoscaling/alloy-metrics.alloy @@ -37,6 +37,7 @@ prometheus.remote_write "prometheus" { } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -240,3 +241,37 @@ cluster_metrics "feature" { prometheus.remote_write.prometheus.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/autoscaling/output.yaml b/charts/k8s-monitoring/docs/examples/autoscaling/output.yaml index 472580039d..8db86cf9a3 100644 --- a/charts/k8s-monitoring/docs/examples/autoscaling/output.yaml +++ b/charts/k8s-monitoring/docs/examples/autoscaling/output.yaml @@ -88,7 +88,7 @@ data: service: services-where: "Name='containerd' or Name='kubelet'" --- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -135,6 +135,7 @@ data: } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -338,6 +339,49 @@ data: prometheus.remote_write.prometheus.receiver, ] } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{deployments="kube-state-metrics,node-exporter,windows-exporter", feature="clusterMetrics", sources="kubelet,kubeletResource,cadvisor,kube-state-metrics,node-exporter,windows-exporter", version="1.0.0"} 1 --- # Source: k8s-monitoring/charts/alloy-metrics/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/collector-storage/alloy-logs.alloy b/charts/k8s-monitoring/docs/examples/collector-storage/alloy-logs.alloy index 9d5b03ae10..b15e28299b 100644 --- a/charts/k8s-monitoring/docs/examples/collector-storage/alloy-logs.alloy +++ b/charts/k8s-monitoring/docs/examples/collector-storage/alloy-logs.alloy @@ -12,6 +12,7 @@ loki.write "loki" { "k8s_cluster_name" = "collector-storage-example-cluster", } } + // Feature: Pod Logs declare "pod_logs" { argument "logs_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/collector-storage/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/collector-storage/alloy-metrics.alloy index ee5ea17250..67397785c0 100644 --- a/charts/k8s-monitoring/docs/examples/collector-storage/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/collector-storage/alloy-metrics.alloy @@ -37,6 +37,7 @@ prometheus.remote_write "prometheus" { } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -240,3 +241,37 @@ cluster_metrics "feature" { prometheus.remote_write.prometheus.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/collector-storage/output.yaml b/charts/k8s-monitoring/docs/examples/collector-storage/output.yaml index 364bc92c0d..eeb740325c 100644 --- a/charts/k8s-monitoring/docs/examples/collector-storage/output.yaml +++ b/charts/k8s-monitoring/docs/examples/collector-storage/output.yaml @@ -104,145 +104,7 @@ data: service: services-where: "Name='containerd' or Name='kubelet'" --- -# Source: k8s-monitoring/templates/alloy-logs-config.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: ko-alloy-logs - namespace: default -data: - config.alloy: |- - // Destination: loki (loki) - otelcol.exporter.loki "loki" { - forward_to = [loki.write.loki.receiver] - } - - loki.write "loki" { - endpoint { - url = "http://loki.loki.svc:3100/loki/api/v1/push" - } - external_labels = { - cluster = "collector-storage-example-cluster", - "k8s_cluster_name" = "collector-storage-example-cluster", - } - } - // Feature: Pod Logs - declare "pod_logs" { - argument "logs_destinations" { - comment = "Must be a list of log destinations where collected logs should be forwarded to" - } - - discovery.relabel "filtered_pods" { - targets = discovery.kubernetes.pods.targets - rule { - source_labels = ["__meta_kubernetes_namespace"] - action = "replace" - target_label = "namespace" - } - rule { - source_labels = ["__meta_kubernetes_pod_name"] - action = "replace" - target_label = "pod" - } - rule { - source_labels = ["__meta_kubernetes_pod_container_name"] - action = "replace" - target_label = "container" - } - rule { - source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] - separator = "/" - action = "replace" - replacement = "$1" - target_label = "job" - } - - // set the container runtime as a label - rule { - action = "replace" - source_labels = ["__meta_kubernetes_pod_container_id"] - regex = "^(\\S+):\\/\\/.+$" - replacement = "$1" - target_label = "tmp_container_runtime" - } - } - - discovery.kubernetes "pods" { - role = "pod" - selectors { - role = "pod" - field = "spec.nodeName=" + env("HOSTNAME") - } - } - - discovery.relabel "filtered_pods_with_paths" { - targets = discovery.relabel.filtered_pods.output - - rule { - source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] - separator = "/" - action = "replace" - replacement = "/var/log/pods/*$1/*.log" - target_label = "__path__" - } - } - - local.file_match "pod_logs" { - path_targets = discovery.relabel.filtered_pods_with_paths.output - } - - loki.source.file "pod_logs" { - targets = local.file_match.pod_logs.targets - forward_to = [loki.process.pod_logs.receiver] - } - - loki.process "pod_logs" { - stage.match { - selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" - // the cri processing stage extracts the following k/v pairs: log, stream, time, flags - stage.cri {} - - // Set the extract flags and stream values as labels - stage.labels { - values = { - flags = "", - stream = "", - } - } - } - - stage.match { - selector = "{tmp_container_runtime=\"docker\"}" - // the docker processing stage extracts the following k/v pairs: log, stream, time - stage.docker {} - - // Set the extract stream value as a label - stage.labels { - values = { - stream = "", - } - } - } - - // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, - // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary - // container runtime label as it is no longer needed. - stage.label_drop { - values = [ - "filename", - "tmp_container_runtime", - ] - } - forward_to = argument.logs_destinations.value - } - } - pod_logs "feature" { - logs_destinations = [ - loki.write.loki.receiver, - ] - } ---- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -289,6 +151,7 @@ data: } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -492,6 +355,189 @@ data: prometheus.remote_write.prometheus.receiver, ] } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{deployments="kube-state-metrics,node-exporter,windows-exporter", feature="clusterMetrics", sources="kubelet,kubeletResource,cadvisor,kube-state-metrics,node-exporter,windows-exporter", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="podLogs", method="volumes", version="1.0.0"} 1 +--- +# Source: k8s-monitoring/templates/alloy-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: ko-alloy-logs + namespace: default +data: + config.alloy: |- + // Destination: loki (loki) + otelcol.exporter.loki "loki" { + forward_to = [loki.write.loki.receiver] + } + + loki.write "loki" { + endpoint { + url = "http://loki.loki.svc:3100/loki/api/v1/push" + } + external_labels = { + cluster = "collector-storage-example-cluster", + "k8s_cluster_name" = "collector-storage-example-cluster", + } + } + + // Feature: Pod Logs + declare "pod_logs" { + argument "logs_destinations" { + comment = "Must be a list of log destinations where collected logs should be forwarded to" + } + + discovery.relabel "filtered_pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "$1" + target_label = "job" + } + + // set the container runtime as a label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_id"] + regex = "^(\\S+):\\/\\/.+$" + replacement = "$1" + target_label = "tmp_container_runtime" + } + } + + discovery.kubernetes "pods" { + role = "pod" + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } + } + + discovery.relabel "filtered_pods_with_paths" { + targets = discovery.relabel.filtered_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "/var/log/pods/*$1/*.log" + target_label = "__path__" + } + } + + local.file_match "pod_logs" { + path_targets = discovery.relabel.filtered_pods_with_paths.output + } + + loki.source.file "pod_logs" { + targets = local.file_match.pod_logs.targets + forward_to = [loki.process.pod_logs.receiver] + } + + loki.process "pod_logs" { + stage.match { + selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" + // the cri processing stage extracts the following k/v pairs: log, stream, time, flags + stage.cri {} + + // Set the extract flags and stream values as labels + stage.labels { + values = { + flags = "", + stream = "", + } + } + } + + stage.match { + selector = "{tmp_container_runtime=\"docker\"}" + // the docker processing stage extracts the following k/v pairs: log, stream, time + stage.docker {} + + // Set the extract stream value as a label + stage.labels { + values = { + stream = "", + } + } + } + + // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, + // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary + // container runtime label as it is no longer needed. + stage.label_drop { + values = [ + "filename", + "tmp_container_runtime", + ] + } + forward_to = argument.logs_destinations.value + } + } + pod_logs "feature" { + logs_destinations = [ + loki.write.loki.receiver, + ] + } --- # Source: k8s-monitoring/charts/alloy-logs/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/extra-rules/alloy-logs.alloy b/charts/k8s-monitoring/docs/examples/extra-rules/alloy-logs.alloy index 3e60e21aaf..64b31d8bbb 100644 --- a/charts/k8s-monitoring/docs/examples/extra-rules/alloy-logs.alloy +++ b/charts/k8s-monitoring/docs/examples/extra-rules/alloy-logs.alloy @@ -14,6 +14,7 @@ loki.write "loki" { region = env("REGION"), } } + // Feature: Pod Logs declare "pod_logs" { argument "logs_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/extra-rules/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/extra-rules/alloy-metrics.alloy index 21897f45aa..3bb8763d50 100644 --- a/charts/k8s-monitoring/docs/examples/extra-rules/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/extra-rules/alloy-metrics.alloy @@ -46,6 +46,7 @@ prometheus.remote_write "prometheus" { region = env("REGION"), } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/extra-rules/alloy-singleton.alloy b/charts/k8s-monitoring/docs/examples/extra-rules/alloy-singleton.alloy index fd718e8e2f..5ee847fe7c 100644 --- a/charts/k8s-monitoring/docs/examples/extra-rules/alloy-singleton.alloy +++ b/charts/k8s-monitoring/docs/examples/extra-rules/alloy-singleton.alloy @@ -1,3 +1,51 @@ +// Destination: prometheus (prometheus) +otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] +} + +prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + } + tls_config { + insecure_skip_verify = false + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "extra-rules-example-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "extra-rules-example-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["__name__"] + regex = "metric_to_drop|another_metric_to_drop" + action = "drop" + } + } + external_labels = { + site = "lab2", + region = env("REGION"), + } +} // Destination: loki (loki) otelcol.exporter.loki "loki" { forward_to = [loki.write.loki.receiver] @@ -61,3 +109,37 @@ cluster_events "feature" { loki.write.loki.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/extra-rules/output.yaml b/charts/k8s-monitoring/docs/examples/extra-rules/output.yaml index 3ed0172b93..c25b0428f4 100644 --- a/charts/k8s-monitoring/docs/examples/extra-rules/output.yaml +++ b/charts/k8s-monitoring/docs/examples/extra-rules/output.yaml @@ -120,152 +120,7 @@ data: service: services-where: "Name='containerd' or Name='kubelet'" --- -# Source: k8s-monitoring/templates/alloy-logs-config.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: ko-alloy-logs - namespace: default -data: - config.alloy: |- - // Destination: loki (loki) - otelcol.exporter.loki "loki" { - forward_to = [loki.write.loki.receiver] - } - - loki.write "loki" { - endpoint { - url = "http://loki.loki.svc:3100/loki/api/v1/push" - } - external_labels = { - cluster = "extra-rules-example-cluster", - "k8s_cluster_name" = "extra-rules-example-cluster", - site = "lab2", - region = env("REGION"), - } - } - // Feature: Pod Logs - declare "pod_logs" { - argument "logs_destinations" { - comment = "Must be a list of log destinations where collected logs should be forwarded to" - } - - discovery.relabel "filtered_pods" { - targets = discovery.kubernetes.pods.targets - rule { - source_labels = ["__meta_kubernetes_namespace"] - action = "replace" - target_label = "namespace" - } - rule { - source_labels = ["__meta_kubernetes_pod_name"] - action = "replace" - target_label = "pod" - } - rule { - source_labels = ["__meta_kubernetes_pod_container_name"] - action = "replace" - target_label = "container" - } - rule { - source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] - separator = "/" - action = "replace" - replacement = "$1" - target_label = "job" - } - - // set the container runtime as a label - rule { - action = "replace" - source_labels = ["__meta_kubernetes_pod_container_id"] - regex = "^(\\S+):\\/\\/.+$" - replacement = "$1" - target_label = "tmp_container_runtime" - } - rule { - source_labels = ["__meta_kubernetes_namespace"] - regex = "production" - action = "keep" - } - } - - discovery.kubernetes "pods" { - role = "pod" - selectors { - role = "pod" - field = "spec.nodeName=" + env("HOSTNAME") - } - } - - discovery.relabel "filtered_pods_with_paths" { - targets = discovery.relabel.filtered_pods.output - - rule { - source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] - separator = "/" - action = "replace" - replacement = "/var/log/pods/*$1/*.log" - target_label = "__path__" - } - } - - local.file_match "pod_logs" { - path_targets = discovery.relabel.filtered_pods_with_paths.output - } - - loki.source.file "pod_logs" { - targets = local.file_match.pod_logs.targets - forward_to = [loki.process.pod_logs.receiver] - } - - loki.process "pod_logs" { - stage.match { - selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" - // the cri processing stage extracts the following k/v pairs: log, stream, time, flags - stage.cri {} - - // Set the extract flags and stream values as labels - stage.labels { - values = { - flags = "", - stream = "", - } - } - } - - stage.match { - selector = "{tmp_container_runtime=\"docker\"}" - // the docker processing stage extracts the following k/v pairs: log, stream, time - stage.docker {} - - // Set the extract stream value as a label - stage.labels { - values = { - stream = "", - } - } - } - - // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, - // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary - // container runtime label as it is no longer needed. - stage.label_drop { - values = [ - "filename", - "tmp_container_runtime", - ] - } - forward_to = argument.logs_destinations.value - } - } - pod_logs "feature" { - logs_destinations = [ - loki.write.loki.receiver, - ] - } ---- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -321,6 +176,7 @@ data: region = env("REGION"), } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -525,7 +381,7 @@ data: ] } --- -# Source: k8s-monitoring/templates/alloy-singleton-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -533,6 +389,54 @@ metadata: namespace: default data: config.alloy: |- + // Destination: prometheus (prometheus) + otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] + } + + prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + } + tls_config { + insecure_skip_verify = false + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "extra-rules-example-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "extra-rules-example-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["__name__"] + regex = "metric_to_drop|another_metric_to_drop" + action = "drop" + } + } + external_labels = { + site = "lab2", + region = env("REGION"), + } + } // Destination: loki (loki) otelcol.exporter.loki "loki" { forward_to = [loki.write.loki.receiver] @@ -596,6 +500,197 @@ data: loki.write.loki.receiver, ] } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{deployments="kube-state-metrics,node-exporter,windows-exporter", feature="clusterMetrics", sources="kubelet,kubeletResource,cadvisor,kube-state-metrics,node-exporter,windows-exporter", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="clusterEvents", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="podLogs", method="volumes", version="1.0.0"} 1 +--- +# Source: k8s-monitoring/templates/alloy-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: ko-alloy-logs + namespace: default +data: + config.alloy: |- + // Destination: loki (loki) + otelcol.exporter.loki "loki" { + forward_to = [loki.write.loki.receiver] + } + + loki.write "loki" { + endpoint { + url = "http://loki.loki.svc:3100/loki/api/v1/push" + } + external_labels = { + cluster = "extra-rules-example-cluster", + "k8s_cluster_name" = "extra-rules-example-cluster", + site = "lab2", + region = env("REGION"), + } + } + + // Feature: Pod Logs + declare "pod_logs" { + argument "logs_destinations" { + comment = "Must be a list of log destinations where collected logs should be forwarded to" + } + + discovery.relabel "filtered_pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "$1" + target_label = "job" + } + + // set the container runtime as a label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_id"] + regex = "^(\\S+):\\/\\/.+$" + replacement = "$1" + target_label = "tmp_container_runtime" + } + rule { + source_labels = ["__meta_kubernetes_namespace"] + regex = "production" + action = "keep" + } + } + + discovery.kubernetes "pods" { + role = "pod" + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } + } + + discovery.relabel "filtered_pods_with_paths" { + targets = discovery.relabel.filtered_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "/var/log/pods/*$1/*.log" + target_label = "__path__" + } + } + + local.file_match "pod_logs" { + path_targets = discovery.relabel.filtered_pods_with_paths.output + } + + loki.source.file "pod_logs" { + targets = local.file_match.pod_logs.targets + forward_to = [loki.process.pod_logs.receiver] + } + + loki.process "pod_logs" { + stage.match { + selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" + // the cri processing stage extracts the following k/v pairs: log, stream, time, flags + stage.cri {} + + // Set the extract flags and stream values as labels + stage.labels { + values = { + flags = "", + stream = "", + } + } + } + + stage.match { + selector = "{tmp_container_runtime=\"docker\"}" + // the docker processing stage extracts the following k/v pairs: log, stream, time + stage.docker {} + + // Set the extract stream value as a label + stage.labels { + values = { + stream = "", + } + } + } + + // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, + // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary + // container runtime label as it is no longer needed. + stage.label_drop { + values = [ + "filename", + "tmp_container_runtime", + ] + } + forward_to = argument.logs_destinations.value + } + } + pod_logs "feature" { + logs_destinations = [ + loki.write.loki.receiver, + ] + } --- # Source: k8s-monitoring/charts/alloy-logs/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/features/annotation-autodiscovery/default/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/features/annotation-autodiscovery/default/alloy-metrics.alloy index 4dad6a23de..d84022c876 100644 --- a/charts/k8s-monitoring/docs/examples/features/annotation-autodiscovery/default/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/features/annotation-autodiscovery/default/alloy-metrics.alloy @@ -238,3 +238,37 @@ annotation_autodiscovery "feature" { prometheus.remote_write.prometheus.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/features/annotation-autodiscovery/default/output.yaml b/charts/k8s-monitoring/docs/examples/features/annotation-autodiscovery/default/output.yaml index 82b24b3fb3..29a94de9d9 100644 --- a/charts/k8s-monitoring/docs/examples/features/annotation-autodiscovery/default/output.yaml +++ b/charts/k8s-monitoring/docs/examples/features/annotation-autodiscovery/default/output.yaml @@ -15,7 +15,7 @@ metadata: app.kubernetes.io/part-of: alloy app.kubernetes.io/component: rbac --- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -263,6 +263,49 @@ data: prometheus.remote_write.prometheus.receiver, ] } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{feature="annotationAutodiscovery", version="1.0.0"} 1 --- # Source: k8s-monitoring/charts/alloy-metrics/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/features/annotation-autodiscovery/prom-annotations/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/features/annotation-autodiscovery/prom-annotations/alloy-metrics.alloy index a3341e28e6..7e0f492dd9 100644 --- a/charts/k8s-monitoring/docs/examples/features/annotation-autodiscovery/prom-annotations/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/features/annotation-autodiscovery/prom-annotations/alloy-metrics.alloy @@ -238,3 +238,37 @@ annotation_autodiscovery "feature" { prometheus.remote_write.prometheus.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/features/annotation-autodiscovery/prom-annotations/output.yaml b/charts/k8s-monitoring/docs/examples/features/annotation-autodiscovery/prom-annotations/output.yaml index fe96352244..65553c8b05 100644 --- a/charts/k8s-monitoring/docs/examples/features/annotation-autodiscovery/prom-annotations/output.yaml +++ b/charts/k8s-monitoring/docs/examples/features/annotation-autodiscovery/prom-annotations/output.yaml @@ -15,7 +15,7 @@ metadata: app.kubernetes.io/part-of: alloy app.kubernetes.io/component: rbac --- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -263,6 +263,49 @@ data: prometheus.remote_write.prometheus.receiver, ] } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{feature="annotationAutodiscovery", version="1.0.0"} 1 --- # Source: k8s-monitoring/charts/alloy-metrics/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/features/applicaiton-observability/default/alloy-receiver.alloy b/charts/k8s-monitoring/docs/examples/features/applicaiton-observability/default/alloy-receiver.alloy index 8c22014ca7..736dddde59 100644 --- a/charts/k8s-monitoring/docs/examples/features/applicaiton-observability/default/alloy-receiver.alloy +++ b/charts/k8s-monitoring/docs/examples/features/applicaiton-observability/default/alloy-receiver.alloy @@ -43,7 +43,6 @@ otelcol.exporter.otlp "otlp_gateway" { } } - // Feature: Application Observability declare "application_observability" { argument "metrics_destinations" { @@ -154,3 +153,37 @@ application_observability "feature" { otelcol.processor.transform.otlp_gateway.input, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + otelcol.receiver.prometheus.otlp_gateway.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/features/applicaiton-observability/default/output.yaml b/charts/k8s-monitoring/docs/examples/features/applicaiton-observability/default/output.yaml index 1b02f99341..43f2b39666 100644 --- a/charts/k8s-monitoring/docs/examples/features/applicaiton-observability/default/output.yaml +++ b/charts/k8s-monitoring/docs/examples/features/applicaiton-observability/default/output.yaml @@ -15,7 +15,7 @@ metadata: app.kubernetes.io/part-of: alloy app.kubernetes.io/component: rbac --- -# Source: k8s-monitoring/templates/alloy-receiver-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -68,7 +68,6 @@ data: } } - // Feature: Application Observability declare "application_observability" { argument "metrics_destinations" { @@ -179,6 +178,49 @@ data: otelcol.processor.transform.otlp_gateway.input, ] } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + otelcol.receiver.prometheus.otlp_gateway.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{feature="applicationObservability", protocols="otlpgrpc", version="1.0.0"} 1 --- # Source: k8s-monitoring/charts/alloy-receiver/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/features/cluster-events/default/output.yaml b/charts/k8s-monitoring/docs/examples/features/cluster-events/default/output.yaml index a27248dfb2..701b64f070 100644 --- a/charts/k8s-monitoring/docs/examples/features/cluster-events/default/output.yaml +++ b/charts/k8s-monitoring/docs/examples/features/cluster-events/default/output.yaml @@ -15,7 +15,7 @@ metadata: app.kubernetes.io/part-of: alloy app.kubernetes.io/component: rbac --- -# Source: k8s-monitoring/templates/alloy-singleton-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/k8s-monitoring/docs/examples/features/cluster-metrics/control-plane-monitoring/alloy-logs.alloy b/charts/k8s-monitoring/docs/examples/features/cluster-metrics/control-plane-monitoring/alloy-logs.alloy index dd49bf6fba..a00d4835a8 100644 --- a/charts/k8s-monitoring/docs/examples/features/cluster-metrics/control-plane-monitoring/alloy-logs.alloy +++ b/charts/k8s-monitoring/docs/examples/features/cluster-metrics/control-plane-monitoring/alloy-logs.alloy @@ -12,6 +12,7 @@ loki.write "loki" { "k8s_cluster_name" = "cluster-metrics-cluster", } } + // Feature: Pod Logs declare "pod_logs" { argument "logs_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/features/cluster-metrics/control-plane-monitoring/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/features/cluster-metrics/control-plane-monitoring/alloy-metrics.alloy index c1a95b4cab..ea5af2ad9d 100644 --- a/charts/k8s-monitoring/docs/examples/features/cluster-metrics/control-plane-monitoring/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/features/cluster-metrics/control-plane-monitoring/alloy-metrics.alloy @@ -37,6 +37,7 @@ prometheus.remote_write "prometheus" { } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -348,35 +349,3 @@ cluster_metrics "feature" { prometheus.remote_write.prometheus.receiver, ] } - - -declare "etcd_integration" { - argument "metrics_destinations" { - comment = "Must be a list of metric destinations where collected metrics should be forwarded to" - } - - import.git "etcd" { - repository = "https://github.com/grafana/alloy-modules.git" - revision = "main" - path = "modules/databases/kv/etcd/metrics.alloy" - pull_frequency = "15m" - } - - etcd.kubernetes "k8s_controlplane_etcd" { - label_selectors = ["app.kubernetes.io/component=etcd"] - port_name = "metrics" - } - - etcd.scrape "k8s_controlplane_etcd" { - targets = etcd.kubernetes.k8s_controlplane_etcd.output - clustering = true - scrape_interval = "60s" - max_cache_size = 100000 - forward_to = argument.metrics_destinations.value - } -} -etcd_integration "integration" { - metrics_destinations = [ - prometheus.remote_write.prometheus.receiver, - ] -} diff --git a/charts/k8s-monitoring/docs/examples/features/cluster-metrics/control-plane-monitoring/alloy-singleton.alloy b/charts/k8s-monitoring/docs/examples/features/cluster-metrics/control-plane-monitoring/alloy-singleton.alloy index a3a1b7acea..f88b4d971e 100644 --- a/charts/k8s-monitoring/docs/examples/features/cluster-metrics/control-plane-monitoring/alloy-singleton.alloy +++ b/charts/k8s-monitoring/docs/examples/features/cluster-metrics/control-plane-monitoring/alloy-singleton.alloy @@ -1,3 +1,42 @@ +// Destination: prometheus (prometheus) +otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] +} + +prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + } + tls_config { + insecure_skip_verify = false + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "cluster-metrics-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "cluster-metrics-cluster" + target_label = "cluster" + } + } +} // Destination: loki (loki) otelcol.exporter.loki "loki" { forward_to = [loki.write.loki.receiver] @@ -30,3 +69,37 @@ cluster_events "feature" { loki.write.loki.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/features/cluster-metrics/control-plane-monitoring/output.yaml b/charts/k8s-monitoring/docs/examples/features/cluster-metrics/control-plane-monitoring/output.yaml index 84573223be..a6d974ac9e 100644 --- a/charts/k8s-monitoring/docs/examples/features/cluster-metrics/control-plane-monitoring/output.yaml +++ b/charts/k8s-monitoring/docs/examples/features/cluster-metrics/control-plane-monitoring/output.yaml @@ -120,145 +120,7 @@ data: service: services-where: "Name='containerd' or Name='kubelet'" --- -# Source: k8s-monitoring/templates/alloy-logs-config.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: ko-alloy-logs - namespace: default -data: - config.alloy: |- - // Destination: loki (loki) - otelcol.exporter.loki "loki" { - forward_to = [loki.write.loki.receiver] - } - - loki.write "loki" { - endpoint { - url = "http://loki.loki.svc:3100/api/push" - } - external_labels = { - cluster = "cluster-metrics-cluster", - "k8s_cluster_name" = "cluster-metrics-cluster", - } - } - // Feature: Pod Logs - declare "pod_logs" { - argument "logs_destinations" { - comment = "Must be a list of log destinations where collected logs should be forwarded to" - } - - discovery.relabel "filtered_pods" { - targets = discovery.kubernetes.pods.targets - rule { - source_labels = ["__meta_kubernetes_namespace"] - action = "replace" - target_label = "namespace" - } - rule { - source_labels = ["__meta_kubernetes_pod_name"] - action = "replace" - target_label = "pod" - } - rule { - source_labels = ["__meta_kubernetes_pod_container_name"] - action = "replace" - target_label = "container" - } - rule { - source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] - separator = "/" - action = "replace" - replacement = "$1" - target_label = "job" - } - - // set the container runtime as a label - rule { - action = "replace" - source_labels = ["__meta_kubernetes_pod_container_id"] - regex = "^(\\S+):\\/\\/.+$" - replacement = "$1" - target_label = "tmp_container_runtime" - } - } - - discovery.kubernetes "pods" { - role = "pod" - selectors { - role = "pod" - field = "spec.nodeName=" + env("HOSTNAME") - } - } - - discovery.relabel "filtered_pods_with_paths" { - targets = discovery.relabel.filtered_pods.output - - rule { - source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] - separator = "/" - action = "replace" - replacement = "/var/log/pods/*$1/*.log" - target_label = "__path__" - } - } - - local.file_match "pod_logs" { - path_targets = discovery.relabel.filtered_pods_with_paths.output - } - - loki.source.file "pod_logs" { - targets = local.file_match.pod_logs.targets - forward_to = [loki.process.pod_logs.receiver] - } - - loki.process "pod_logs" { - stage.match { - selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" - // the cri processing stage extracts the following k/v pairs: log, stream, time, flags - stage.cri {} - - // Set the extract flags and stream values as labels - stage.labels { - values = { - flags = "", - stream = "", - } - } - } - - stage.match { - selector = "{tmp_container_runtime=\"docker\"}" - // the docker processing stage extracts the following k/v pairs: log, stream, time - stage.docker {} - - // Set the extract stream value as a label - stage.labels { - values = { - stream = "", - } - } - } - - // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, - // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary - // container runtime label as it is no longer needed. - stage.label_drop { - values = [ - "filename", - "tmp_container_runtime", - ] - } - forward_to = argument.logs_destinations.value - } - } - pod_logs "feature" { - logs_destinations = [ - loki.write.loki.receiver, - ] - } ---- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -305,6 +167,7 @@ data: } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -616,44 +479,138 @@ data: prometheus.remote_write.prometheus.receiver, ] } +--- +# Source: k8s-monitoring/templates/alloy-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: ko-alloy-singleton + namespace: default +data: + config.alloy: |- + // Destination: prometheus (prometheus) + otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] + } + + prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + } + tls_config { + insecure_skip_verify = false + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "cluster-metrics-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "cluster-metrics-cluster" + target_label = "cluster" + } + } + } + // Destination: loki (loki) + otelcol.exporter.loki "loki" { + forward_to = [loki.write.loki.receiver] + } + loki.write "loki" { + endpoint { + url = "http://loki.loki.svc:3100/api/push" + } + external_labels = { + cluster = "cluster-metrics-cluster", + "k8s_cluster_name" = "cluster-metrics-cluster", + } + } - declare "etcd_integration" { - argument "metrics_destinations" { - comment = "Must be a list of metric destinations where collected metrics should be forwarded to" + // Feature: Cluster Events + declare "cluster_events" { + argument "logs_destinations" { + comment = "Must be a list of log destinations where collected logs should be forwarded to" } - import.git "etcd" { - repository = "https://github.com/grafana/alloy-modules.git" - revision = "main" - path = "modules/databases/kv/etcd/metrics.alloy" - pull_frequency = "15m" + loki.source.kubernetes_events "cluster_events" { + job_name = "integrations/kubernetes/eventhandler" + log_format = "logfmt" + forward_to = argument.logs_destinations.value } - - etcd.kubernetes "k8s_controlplane_etcd" { - label_selectors = ["app.kubernetes.io/component=etcd"] - port_name = "metrics" + } + cluster_events "feature" { + logs_destinations = [ + loki.write.loki.receiver, + ] + } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" } - - etcd.scrape "k8s_controlplane_etcd" { - targets = etcd.kubernetes.k8s_controlplane_etcd.output - clustering = true - scrape_interval = "60s" - max_cache_size = 100000 - forward_to = argument.metrics_destinations.value + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" } } - etcd_integration "integration" { - metrics_destinations = [ + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ prometheus.remote_write.prometheus.receiver, ] } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{deployments="kube-state-metrics,node-exporter,windows-exporter", feature="clusterMetrics", sources="controlPlane,kubelet,kubeletResource,cadvisor,kube-state-metrics,node-exporter,windows-exporter", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="clusterEvents", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="podLogs", method="volumes", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="integrations", version="1.0.0"} 1 --- -# Source: k8s-monitoring/templates/alloy-singleton-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: - name: ko-alloy-singleton + name: ko-alloy-logs namespace: default data: config.alloy: |- @@ -672,19 +629,117 @@ data: } } - // Feature: Cluster Events - declare "cluster_events" { + // Feature: Pod Logs + declare "pod_logs" { argument "logs_destinations" { comment = "Must be a list of log destinations where collected logs should be forwarded to" } - - loki.source.kubernetes_events "cluster_events" { - job_name = "integrations/kubernetes/eventhandler" - log_format = "logfmt" + + discovery.relabel "filtered_pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "$1" + target_label = "job" + } + + // set the container runtime as a label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_id"] + regex = "^(\\S+):\\/\\/.+$" + replacement = "$1" + target_label = "tmp_container_runtime" + } + } + + discovery.kubernetes "pods" { + role = "pod" + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } + } + + discovery.relabel "filtered_pods_with_paths" { + targets = discovery.relabel.filtered_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "/var/log/pods/*$1/*.log" + target_label = "__path__" + } + } + + local.file_match "pod_logs" { + path_targets = discovery.relabel.filtered_pods_with_paths.output + } + + loki.source.file "pod_logs" { + targets = local.file_match.pod_logs.targets + forward_to = [loki.process.pod_logs.receiver] + } + + loki.process "pod_logs" { + stage.match { + selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" + // the cri processing stage extracts the following k/v pairs: log, stream, time, flags + stage.cri {} + + // Set the extract flags and stream values as labels + stage.labels { + values = { + flags = "", + stream = "", + } + } + } + + stage.match { + selector = "{tmp_container_runtime=\"docker\"}" + // the docker processing stage extracts the following k/v pairs: log, stream, time + stage.docker {} + + // Set the extract stream value as a label + stage.labels { + values = { + stream = "", + } + } + } + + // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, + // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary + // container runtime label as it is no longer needed. + stage.label_drop { + values = [ + "filename", + "tmp_container_runtime", + ] + } forward_to = argument.logs_destinations.value } } - cluster_events "feature" { + pod_logs "feature" { logs_destinations = [ loki.write.loki.receiver, ] diff --git a/charts/k8s-monitoring/docs/examples/features/cluster-metrics/default/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/features/cluster-metrics/default/alloy-metrics.alloy index 188a14dc7f..83a48866f8 100644 --- a/charts/k8s-monitoring/docs/examples/features/cluster-metrics/default/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/features/cluster-metrics/default/alloy-metrics.alloy @@ -37,6 +37,7 @@ prometheus.remote_write "prometheus" { } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -240,3 +241,37 @@ cluster_metrics "feature" { prometheus.remote_write.prometheus.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/features/cluster-metrics/default/output.yaml b/charts/k8s-monitoring/docs/examples/features/cluster-metrics/default/output.yaml index 04768189da..9f10a65f88 100644 --- a/charts/k8s-monitoring/docs/examples/features/cluster-metrics/default/output.yaml +++ b/charts/k8s-monitoring/docs/examples/features/cluster-metrics/default/output.yaml @@ -88,7 +88,7 @@ data: service: services-where: "Name='containerd' or Name='kubelet'" --- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -135,6 +135,7 @@ data: } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -338,6 +339,49 @@ data: prometheus.remote_write.prometheus.receiver, ] } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{deployments="kube-state-metrics,node-exporter,windows-exporter", feature="clusterMetrics", sources="kubelet,kubeletResource,cadvisor,kube-state-metrics,node-exporter,windows-exporter", version="1.0.0"} 1 --- # Source: k8s-monitoring/charts/alloy-metrics/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/features/integrations/alloy/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/features/integrations/alloy/alloy-metrics.alloy index 0048aede86..fcba4ba6e5 100644 --- a/charts/k8s-monitoring/docs/examples/features/integrations/alloy/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/features/integrations/alloy/alloy-metrics.alloy @@ -38,295 +38,36 @@ prometheus.remote_write "prometheus" { } } - - -declare "alloy_integration" { - argument "metrics_destinations" { - comment = "Must be a list of metric destinations where collected metrics should be forwarded to" +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" } +} - declare "alloy_integration_discovery" { - argument "namespaces" { - comment = "The namespaces to look for targets in (default: [] is all namespaces)" - optional = true - } - - argument "field_selectors" { - comment = "The field selectors to use to find matching targets (default: [])" - optional = true - } - - argument "label_selectors" { - comment = "The label selectors to use to find matching targets (default: [\"app.kubernetes.io/name=alloy\"])" - optional = true - } - - argument "port_name" { - comment = "The of the port to scrape metrics from (default: http-metrics)" - optional = true - } - - // Alloy service discovery for all of the pods - discovery.kubernetes "alloy_pods" { - role = "pod" - - selectors { - role = "pod" - field = join(coalesce(argument.field_selectors.value, []), ",") - label = join(coalesce(argument.label_selectors.value, ["app.kubernetes.io/name=alloy"]), ",") - } - - namespaces { - names = coalesce(argument.namespaces.value, []) - } - } - - // alloy relabelings (pre-scrape) - discovery.relabel "alloy_pods" { - targets = discovery.kubernetes.alloy_pods.targets - - // keep only the specified metrics port name, and pods that are Running and ready - rule { - source_labels = [ - "__meta_kubernetes_pod_container_port_name", - "__meta_kubernetes_pod_phase", - "__meta_kubernetes_pod_ready", - "__meta_kubernetes_pod_container_init", - ] - separator = "@" - regex = coalesce(argument.port_name.value, "metrics") + "@Running@true@false" - action = "keep" - } - - rule { - source_labels = ["__meta_kubernetes_namespace"] - target_label = "namespace" - } - - rule { - source_labels = ["__meta_kubernetes_pod_name"] - target_label = "pod" - } - - rule { - source_labels = ["__meta_kubernetes_pod_container_name"] - target_label = "container" - } - - rule { - source_labels = [ - "__meta_kubernetes_pod_controller_kind", - "__meta_kubernetes_pod_controller_name", - ] - separator = "/" - target_label = "workload" - } - // remove the hash from the ReplicaSet - rule { - source_labels = ["workload"] - regex = "(ReplicaSet/.+)-.+" - target_label = "workload" - } - - // set the app name if specified as metadata labels "app:" or "app.kubernetes.io/name:" or "k8s-app:" - rule { - action = "replace" - source_labels = [ - "__meta_kubernetes_pod_label_app_kubernetes_io_name", - "__meta_kubernetes_pod_label_k8s_app", - "__meta_kubernetes_pod_label_app", - ] - separator = ";" - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "app" - } - - // set the component if specified as metadata labels "component:" or "app.kubernetes.io/component:" or "k8s-component:" - rule { - action = "replace" - source_labels = [ - "__meta_kubernetes_pod_label_app_kubernetes_io_component", - "__meta_kubernetes_pod_label_k8s_component", - "__meta_kubernetes_pod_label_component", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "component" - } - - // set a source label - rule { - action = "replace" - replacement = "kubernetes" - target_label = "source" - } - } - - export "output" { - value = discovery.relabel.alloy_pods.output - } - } - - declare "alloy_integration_scrape" { - argument "targets" { - comment = "Must be a list() of targets" - } - - argument "forward_to" { - comment = "Must be a list(MetricsReceiver) where collected logs should be forwarded to" - } - - argument "job_label" { - comment = "The job label to add for all alloy metric (default: integrations/alloy)" - optional = true - } - - argument "keep_metrics" { - comment = "A regular expression of metrics to keep (default: see below)" - optional = true - } - - argument "drop_metrics" { - comment = "A regular expression of metrics to drop (default: see below)" - optional = true - } - - argument "scrape_interval" { - comment = "How often to scrape metrics from the targets (default: 60s)" - optional = true - } - - argument "scrape_timeout" { - comment = "How long before a scrape times out (default: 10s)" - optional = true - } - - argument "max_cache_size" { - comment = "The maximum number of elements to hold in the relabeling cache (default: 100000). This should be at least 2x-5x your largest scrape target or samples appended rate." - optional = true - } - - argument "clustering" { - comment = "Whether or not clustering should be enabled (default: false)" - optional = true - } - - prometheus.scrape "alloy" { - job_name = coalesce(argument.job_label.value, "integrations/alloy") - forward_to = [prometheus.relabel.alloy.receiver] - targets = argument.targets.value - scrape_interval = coalesce(argument.scrape_interval.value, "60s") - scrape_timeout = coalesce(argument.scrape_timeout.value, "10s") - - clustering { - enabled = coalesce(argument.clustering.value, false) - } - } - - // alloy metric relabelings (post-scrape) - prometheus.relabel "alloy" { - forward_to = argument.forward_to.value - max_cache_size = coalesce(argument.max_cache_size.value, 100000) - - // drop metrics that match the drop_metrics regex - rule { - source_labels = ["__name__"] - regex = coalesce(argument.drop_metrics.value, "(^(go|process)_.+$)") - action = "drop" - } - - // keep only metrics that match the keep_metrics regex - rule { - source_labels = ["__name__"] - regex = coalesce(argument.keep_metrics.value, ".*") - action = "keep" - } - - // remove the component_id label from any metric that starts with log_bytes or log_lines, these are custom metrics that are generated - // as part of the log annotation modules in this repo - rule { - action = "replace" - source_labels = ["__name__"] - regex = "^log_(bytes|lines).+" - replacement = "" - target_label = "component_id" - } - - // set the namespace label to that of the exported_namespace - rule { - action = "replace" - source_labels = ["__name__", "exported_namespace"] - separator = "@" - regex = "^log_(bytes|lines).+@(.+)" - replacement = "$2" - target_label = "namespace" - } - - // set the pod label to that of the exported_pod - rule { - action = "replace" - source_labels = ["__name__", "exported_pod"] - separator = "@" - regex = "^log_(bytes|lines).+@(.+)" - replacement = "$2" - target_label = "pod" - } - - // set the container label to that of the exported_container - rule { - action = "replace" - source_labels = ["__name__", "exported_container"] - separator = "@" - regex = "^log_(bytes|lines).+@(.+)" - replacement = "$2" - target_label = "container" - } - - // set the job label to that of the exported_job - rule { - action = "replace" - source_labels = ["__name__", "exported_job"] - separator = "@" - regex = "^log_(bytes|lines).+@(.+)" - replacement = "$2" - target_label = "job" - } - - // set the instance label to that of the exported_instance - rule { - action = "replace" - source_labels = ["__name__", "exported_instance"] - separator = "@" - regex = "^log_(bytes|lines).+@(.+)" - replacement = "$2" - target_label = "instance" - } - - rule { - action = "labeldrop" - regex = "exported_(namespace|pod|container|job|instance)" - } - } +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" } - - - alloy_integration_discovery "alloy_metrics" { - port_name = "http-metrics" - label_selectors = ["app.kubernetes.io/name=alloy-metrics"] - } - - alloy_integration_scrape "alloy_metrics" { - targets = alloy_integration_discovery.alloy_metrics.output - clustering = true - keep_metrics = "up|alloy_build_info" - scrape_interval = "60s" - max_cache_size = 100000 - forward_to = argument.metrics_destinations.value + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" } } -alloy_integration "integration" { - metrics_destinations = [ + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ prometheus.remote_write.prometheus.receiver, ] } diff --git a/charts/k8s-monitoring/docs/examples/features/integrations/alloy/output.yaml b/charts/k8s-monitoring/docs/examples/features/integrations/alloy/output.yaml index fad9be3156..9df977be7c 100644 --- a/charts/k8s-monitoring/docs/examples/features/integrations/alloy/output.yaml +++ b/charts/k8s-monitoring/docs/examples/features/integrations/alloy/output.yaml @@ -15,7 +15,7 @@ metadata: app.kubernetes.io/part-of: alloy app.kubernetes.io/component: rbac --- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -63,298 +63,48 @@ data: } } - - - declare "alloy_integration" { - argument "metrics_destinations" { - comment = "Must be a list of metric destinations where collected metrics should be forwarded to" - } - - declare "alloy_integration_discovery" { - argument "namespaces" { - comment = "The namespaces to look for targets in (default: [] is all namespaces)" - optional = true - } - - argument "field_selectors" { - comment = "The field selectors to use to find matching targets (default: [])" - optional = true - } - - argument "label_selectors" { - comment = "The label selectors to use to find matching targets (default: [\"app.kubernetes.io/name=alloy\"])" - optional = true - } - - argument "port_name" { - comment = "The of the port to scrape metrics from (default: http-metrics)" - optional = true - } - - // Alloy service discovery for all of the pods - discovery.kubernetes "alloy_pods" { - role = "pod" - - selectors { - role = "pod" - field = join(coalesce(argument.field_selectors.value, []), ",") - label = join(coalesce(argument.label_selectors.value, ["app.kubernetes.io/name=alloy"]), ",") - } - - namespaces { - names = coalesce(argument.namespaces.value, []) - } - } - - // alloy relabelings (pre-scrape) - discovery.relabel "alloy_pods" { - targets = discovery.kubernetes.alloy_pods.targets - - // keep only the specified metrics port name, and pods that are Running and ready - rule { - source_labels = [ - "__meta_kubernetes_pod_container_port_name", - "__meta_kubernetes_pod_phase", - "__meta_kubernetes_pod_ready", - "__meta_kubernetes_pod_container_init", - ] - separator = "@" - regex = coalesce(argument.port_name.value, "metrics") + "@Running@true@false" - action = "keep" - } - - rule { - source_labels = ["__meta_kubernetes_namespace"] - target_label = "namespace" - } - - rule { - source_labels = ["__meta_kubernetes_pod_name"] - target_label = "pod" - } - - rule { - source_labels = ["__meta_kubernetes_pod_container_name"] - target_label = "container" - } - - rule { - source_labels = [ - "__meta_kubernetes_pod_controller_kind", - "__meta_kubernetes_pod_controller_name", - ] - separator = "/" - target_label = "workload" - } - // remove the hash from the ReplicaSet - rule { - source_labels = ["workload"] - regex = "(ReplicaSet/.+)-.+" - target_label = "workload" - } - - // set the app name if specified as metadata labels "app:" or "app.kubernetes.io/name:" or "k8s-app:" - rule { - action = "replace" - source_labels = [ - "__meta_kubernetes_pod_label_app_kubernetes_io_name", - "__meta_kubernetes_pod_label_k8s_app", - "__meta_kubernetes_pod_label_app", - ] - separator = ";" - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "app" - } - - // set the component if specified as metadata labels "component:" or "app.kubernetes.io/component:" or "k8s-component:" - rule { - action = "replace" - source_labels = [ - "__meta_kubernetes_pod_label_app_kubernetes_io_component", - "__meta_kubernetes_pod_label_k8s_component", - "__meta_kubernetes_pod_label_component", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "component" - } - - // set a source label - rule { - action = "replace" - replacement = "kubernetes" - target_label = "source" - } - } - - export "output" { - value = discovery.relabel.alloy_pods.output - } + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" } + } - declare "alloy_integration_scrape" { - argument "targets" { - comment = "Must be a list() of targets" - } - - argument "forward_to" { - comment = "Must be a list(MetricsReceiver) where collected logs should be forwarded to" - } - - argument "job_label" { - comment = "The job label to add for all alloy metric (default: integrations/alloy)" - optional = true - } - - argument "keep_metrics" { - comment = "A regular expression of metrics to keep (default: see below)" - optional = true - } - - argument "drop_metrics" { - comment = "A regular expression of metrics to drop (default: see below)" - optional = true - } - - argument "scrape_interval" { - comment = "How often to scrape metrics from the targets (default: 60s)" - optional = true - } - - argument "scrape_timeout" { - comment = "How long before a scrape times out (default: 10s)" - optional = true - } - - argument "max_cache_size" { - comment = "The maximum number of elements to hold in the relabeling cache (default: 100000). This should be at least 2x-5x your largest scrape target or samples appended rate." - optional = true - } - - argument "clustering" { - comment = "Whether or not clustering should be enabled (default: false)" - optional = true - } - - prometheus.scrape "alloy" { - job_name = coalesce(argument.job_label.value, "integrations/alloy") - forward_to = [prometheus.relabel.alloy.receiver] - targets = argument.targets.value - scrape_interval = coalesce(argument.scrape_interval.value, "60s") - scrape_timeout = coalesce(argument.scrape_timeout.value, "10s") - - clustering { - enabled = coalesce(argument.clustering.value, false) - } - } - - // alloy metric relabelings (post-scrape) - prometheus.relabel "alloy" { - forward_to = argument.forward_to.value - max_cache_size = coalesce(argument.max_cache_size.value, 100000) - - // drop metrics that match the drop_metrics regex - rule { - source_labels = ["__name__"] - regex = coalesce(argument.drop_metrics.value, "(^(go|process)_.+$)") - action = "drop" - } - - // keep only metrics that match the keep_metrics regex - rule { - source_labels = ["__name__"] - regex = coalesce(argument.keep_metrics.value, ".*") - action = "keep" - } - - // remove the component_id label from any metric that starts with log_bytes or log_lines, these are custom metrics that are generated - // as part of the log annotation modules in this repo - rule { - action = "replace" - source_labels = ["__name__"] - regex = "^log_(bytes|lines).+" - replacement = "" - target_label = "component_id" - } - - // set the namespace label to that of the exported_namespace - rule { - action = "replace" - source_labels = ["__name__", "exported_namespace"] - separator = "@" - regex = "^log_(bytes|lines).+@(.+)" - replacement = "$2" - target_label = "namespace" - } - - // set the pod label to that of the exported_pod - rule { - action = "replace" - source_labels = ["__name__", "exported_pod"] - separator = "@" - regex = "^log_(bytes|lines).+@(.+)" - replacement = "$2" - target_label = "pod" - } - - // set the container label to that of the exported_container - rule { - action = "replace" - source_labels = ["__name__", "exported_container"] - separator = "@" - regex = "^log_(bytes|lines).+@(.+)" - replacement = "$2" - target_label = "container" - } - - // set the job label to that of the exported_job - rule { - action = "replace" - source_labels = ["__name__", "exported_job"] - separator = "@" - regex = "^log_(bytes|lines).+@(.+)" - replacement = "$2" - target_label = "job" - } - - // set the instance label to that of the exported_instance - rule { - action = "replace" - source_labels = ["__name__", "exported_instance"] - separator = "@" - regex = "^log_(bytes|lines).+@(.+)" - replacement = "$2" - target_label = "instance" - } - - rule { - action = "labeldrop" - regex = "exported_(namespace|pod|container|job|instance)" - } - } - } - - - alloy_integration_discovery "alloy_metrics" { - port_name = "http-metrics" - label_selectors = ["app.kubernetes.io/name=alloy-metrics"] + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" } - - alloy_integration_scrape "alloy_metrics" { - targets = alloy_integration_discovery.alloy_metrics.output - clustering = true - keep_metrics = "up|alloy_build_info" - scrape_interval = "60s" - max_cache_size = 100000 - forward_to = argument.metrics_destinations.value + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" } } - alloy_integration "integration" { - metrics_destinations = [ + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ prometheus.remote_write.prometheus.receiver, ] } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{feature="integrations", version="1.0.0"} 1 --- # Source: k8s-monitoring/charts/alloy-metrics/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/features/integrations/cert-manager/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/features/integrations/cert-manager/alloy-metrics.alloy index 4ac7584943..ac69f5ae7e 100644 --- a/charts/k8s-monitoring/docs/examples/features/integrations/cert-manager/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/features/integrations/cert-manager/alloy-metrics.alloy @@ -38,35 +38,36 @@ prometheus.remote_write "prometheus" { } } - - -declare "cert_manager_integration" { - argument "metrics_destinations" { - comment = "Must be a list of metric destinations where collected metrics should be forwarded to" +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" } +} - import.git "cert_manager" { - repository = "https://github.com/grafana/alloy-modules.git" - revision = "main" - path = "modules/kubernetes/cert-manager/metrics.alloy" - pull_frequency = "15m" +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" } - - cert_manager.kubernetes "cert_manager" { - label_selectors = ["app.kubernetes.io/name=cert-manager"] - port_name = "http-metrics" - } - - cert_manager.scrape "cert_manager" { - targets = cert_manager.kubernetes.cert_manager.output - clustering = true - scrape_interval = "60s" - max_cache_size = 100000 - forward_to = argument.metrics_destinations.value + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" } } -cert_manager_integration "integration" { - metrics_destinations = [ + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ prometheus.remote_write.prometheus.receiver, ] } diff --git a/charts/k8s-monitoring/docs/examples/features/integrations/cert-manager/output.yaml b/charts/k8s-monitoring/docs/examples/features/integrations/cert-manager/output.yaml index a622c8891c..e20261a193 100644 --- a/charts/k8s-monitoring/docs/examples/features/integrations/cert-manager/output.yaml +++ b/charts/k8s-monitoring/docs/examples/features/integrations/cert-manager/output.yaml @@ -15,7 +15,7 @@ metadata: app.kubernetes.io/part-of: alloy app.kubernetes.io/component: rbac --- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -63,38 +63,48 @@ data: } } - - - declare "cert_manager_integration" { - argument "metrics_destinations" { - comment = "Must be a list of metric destinations where collected metrics should be forwarded to" + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" } + } - import.git "cert_manager" { - repository = "https://github.com/grafana/alloy-modules.git" - revision = "main" - path = "modules/kubernetes/cert-manager/metrics.alloy" - pull_frequency = "15m" - } - - cert_manager.kubernetes "cert_manager" { - label_selectors = ["app.kubernetes.io/name=cert-manager"] - port_name = "http-metrics" + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" } - - cert_manager.scrape "cert_manager" { - targets = cert_manager.kubernetes.cert_manager.output - clustering = true - scrape_interval = "60s" - max_cache_size = 100000 - forward_to = argument.metrics_destinations.value + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" } } - cert_manager_integration "integration" { - metrics_destinations = [ + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ prometheus.remote_write.prometheus.receiver, ] } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{feature="integrations", version="1.0.0"} 1 --- # Source: k8s-monitoring/charts/alloy-metrics/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/features/integrations/etcd/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/features/integrations/etcd/alloy-metrics.alloy index 78e16c97d8..9633768c03 100644 --- a/charts/k8s-monitoring/docs/examples/features/integrations/etcd/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/features/integrations/etcd/alloy-metrics.alloy @@ -38,35 +38,36 @@ prometheus.remote_write "prometheus" { } } - - -declare "etcd_integration" { - argument "metrics_destinations" { - comment = "Must be a list of metric destinations where collected metrics should be forwarded to" +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" } +} - import.git "etcd" { - repository = "https://github.com/grafana/alloy-modules.git" - revision = "main" - path = "modules/databases/kv/etcd/metrics.alloy" - pull_frequency = "15m" +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" } - - etcd.kubernetes "etcd" { - label_selectors = ["app.kubernetes.io/component=etcd"] - port_name = "metrics" - } - - etcd.scrape "etcd" { - targets = etcd.kubernetes.etcd.output - clustering = true - scrape_interval = "60s" - max_cache_size = 100000 - forward_to = argument.metrics_destinations.value + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" } } -etcd_integration "integration" { - metrics_destinations = [ + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ prometheus.remote_write.prometheus.receiver, ] } diff --git a/charts/k8s-monitoring/docs/examples/features/integrations/etcd/output.yaml b/charts/k8s-monitoring/docs/examples/features/integrations/etcd/output.yaml index 3f70bff866..2ecd845cf5 100644 --- a/charts/k8s-monitoring/docs/examples/features/integrations/etcd/output.yaml +++ b/charts/k8s-monitoring/docs/examples/features/integrations/etcd/output.yaml @@ -15,7 +15,7 @@ metadata: app.kubernetes.io/part-of: alloy app.kubernetes.io/component: rbac --- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -63,38 +63,48 @@ data: } } - - - declare "etcd_integration" { - argument "metrics_destinations" { - comment = "Must be a list of metric destinations where collected metrics should be forwarded to" + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" } + } - import.git "etcd" { - repository = "https://github.com/grafana/alloy-modules.git" - revision = "main" - path = "modules/databases/kv/etcd/metrics.alloy" - pull_frequency = "15m" - } - - etcd.kubernetes "etcd" { - label_selectors = ["app.kubernetes.io/component=etcd"] - port_name = "metrics" + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" } - - etcd.scrape "etcd" { - targets = etcd.kubernetes.etcd.output - clustering = true - scrape_interval = "60s" - max_cache_size = 100000 - forward_to = argument.metrics_destinations.value + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" } } - etcd_integration "integration" { - metrics_destinations = [ + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ prometheus.remote_write.prometheus.receiver, ] } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{feature="integrations", version="1.0.0"} 1 --- # Source: k8s-monitoring/charts/alloy-metrics/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/features/pod-logs/default/output.yaml b/charts/k8s-monitoring/docs/examples/features/pod-logs/default/output.yaml index 0dbb7267e3..9278d433a9 100644 --- a/charts/k8s-monitoring/docs/examples/features/pod-logs/default/output.yaml +++ b/charts/k8s-monitoring/docs/examples/features/pod-logs/default/output.yaml @@ -15,7 +15,7 @@ metadata: app.kubernetes.io/part-of: alloy app.kubernetes.io/component: rbac --- -# Source: k8s-monitoring/templates/alloy-singleton-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/k8s-monitoring/docs/examples/features/profiling/default/output.yaml b/charts/k8s-monitoring/docs/examples/features/profiling/default/output.yaml index 401d53e108..1ae8db6fc1 100644 --- a/charts/k8s-monitoring/docs/examples/features/profiling/default/output.yaml +++ b/charts/k8s-monitoring/docs/examples/features/profiling/default/output.yaml @@ -15,7 +15,7 @@ metadata: app.kubernetes.io/part-of: alloy app.kubernetes.io/component: rbac --- -# Source: k8s-monitoring/templates/alloy-profiles-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/k8s-monitoring/docs/examples/features/prometheus-operator-objects/default/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/features/prometheus-operator-objects/default/alloy-metrics.alloy index 906d3064f7..dc6eae9070 100644 --- a/charts/k8s-monitoring/docs/examples/features/prometheus-operator-objects/default/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/features/prometheus-operator-objects/default/alloy-metrics.alloy @@ -38,7 +38,6 @@ prometheus.remote_write "prometheus" { } } - // Feature: Prometheus Operator Objects declare "prometheus_operator_objects" { argument "metrics_destinations" { @@ -83,3 +82,37 @@ prometheus_operator_objects "feature" { prometheus.remote_write.prometheus.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/features/prometheus-operator-objects/default/output.yaml b/charts/k8s-monitoring/docs/examples/features/prometheus-operator-objects/default/output.yaml index 3dd9b65651..0c2a4f6678 100644 --- a/charts/k8s-monitoring/docs/examples/features/prometheus-operator-objects/default/output.yaml +++ b/charts/k8s-monitoring/docs/examples/features/prometheus-operator-objects/default/output.yaml @@ -15,7 +15,7 @@ metadata: app.kubernetes.io/part-of: alloy app.kubernetes.io/component: rbac --- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -63,7 +63,6 @@ data: } } - // Feature: Prometheus Operator Objects declare "prometheus_operator_objects" { argument "metrics_destinations" { @@ -108,6 +107,49 @@ data: prometheus.remote_write.prometheus.receiver, ] } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{feature="prometheusOperatorObjects", version="1.0.0"} 1 --- # Source: k8s-monitoring/charts/alloy-metrics/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/metrics-tuning/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/metrics-tuning/alloy-metrics.alloy index 5c88fbaa64..bed19c69c7 100644 --- a/charts/k8s-monitoring/docs/examples/metrics-tuning/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/metrics-tuning/alloy-metrics.alloy @@ -37,6 +37,218 @@ prometheus.remote_write "prometheus" { } } } + +// Feature: Annotation Autodiscovery +declare "annotation_autodiscovery" { + argument "metrics_destinations" { + comment = "Must be a list of metric destinations where collected metrics should be forwarded to" + } + + discovery.kubernetes "pods" { + role = "pod" + } + + discovery.relabel "annotation_autodiscovery_pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_scrape"] + regex = "true" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_job"] + action = "replace" + target_label = "job" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_instance"] + action = "replace" + target_label = "instance" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_path"] + action = "replace" + target_label = "__metrics_path__" + } + + // Choose the pod port + // The discovery generates a target for each declared container port of the pod. + // If the metricsPortName annotation has value, keep only the target where the port name matches the one of the annotation. + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_portName"] + regex = "(.+)" + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + action = "keepequal" + target_label = "__tmp_port" + } + + // If the metrics port number annotation has a value, override the target address to use it, regardless whether it is + // one of the declared ports on that Pod. + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_portNumber", "__meta_kubernetes_pod_ip"] + regex = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})" + replacement = "[$2]:$1" // IPv6 + target_label = "__address__" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_portNumber", "__meta_kubernetes_pod_ip"] + regex = "(\\d+);((([0-9]+?)(\\.|$)){4})" // IPv4, takes priority over IPv6 when both exists + replacement = "$2:$1" + target_label = "__address__" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_scheme"] + action = "replace" + target_label = "__scheme__" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_scrapeInterval"] + action = "replace" + target_label = "__scrape_interval__" + } + } + + discovery.kubernetes "services" { + role = "service" + } + + discovery.relabel "annotation_autodiscovery_services" { + targets = discovery.kubernetes.services.targets + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_scrape"] + regex = "true" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_job"] + action = "replace" + target_label = "job" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_instance"] + action = "replace" + target_label = "instance" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_path"] + action = "replace" + target_label = "__metrics_path__" + } + + // Choose the service port + rule { + source_labels = ["__meta_kubernetes_service_port_name"] + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_portName"] + regex = "(.+)" + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_service_port_name"] + action = "keepequal" + target_label = "__tmp_port" + } + + rule { + source_labels = ["__meta_kubernetes_service_port_number"] + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_portNumber"] + regex = "(.+)" + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_service_port_number"] + action = "keepequal" + target_label = "__tmp_port" + } + + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_scheme"] + action = "replace" + target_label = "__scheme__" + } + + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_scrapeInterval"] + action = "replace" + target_label = "__scrape_interval__" + } + } + + discovery.relabel "annotation_autodiscovery_http" { + targets = concat(discovery.relabel.annotation_autodiscovery_pods.output, discovery.relabel.annotation_autodiscovery_services.output) + rule { + source_labels = ["__scheme__"] + regex = "https" + action = "drop" + } + } + + discovery.relabel "annotation_autodiscovery_https" { + targets = concat(discovery.relabel.annotation_autodiscovery_pods.output, discovery.relabel.annotation_autodiscovery_services.output) + rule { + source_labels = ["__scheme__"] + regex = "https" + action = "keep" + } + } + + prometheus.scrape "annotation_autodiscovery_http" { + targets = discovery.relabel.annotation_autodiscovery_http.output + scrape_interval = "60s" + honor_labels = true + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + clustering { + enabled = true + } + + forward_to = [prometheus.relabel.annotation_autodiscovery.receiver] + } + + prometheus.scrape "annotation_autodiscovery_https" { + targets = discovery.relabel.annotation_autodiscovery_https.output + scrape_interval = "60s" + honor_labels = true + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + tls_config { + insecure_skip_verify = true + } + clustering { + enabled = true + } + + forward_to = [prometheus.relabel.annotation_autodiscovery.receiver] + } + + prometheus.relabel "annotation_autodiscovery" { + max_cache_size = 100000 + rule { + source_labels = ["__name__"] + regex = "go_*" + action = "drop" + } + forward_to = argument.metrics_destinations.value + } +} +annotation_autodiscovery "feature" { + metrics_destinations = [ + prometheus.remote_write.prometheus.receiver, + ] +} + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -240,213 +452,37 @@ cluster_metrics "feature" { prometheus.remote_write.prometheus.receiver, ] } -// Feature: Annotation Autodiscovery -declare "annotation_autodiscovery" { - argument "metrics_destinations" { - comment = "Must be a list of metric destinations where collected metrics should be forwarded to" - } - - discovery.kubernetes "pods" { - role = "pod" - } - - discovery.relabel "annotation_autodiscovery_pods" { - targets = discovery.kubernetes.pods.targets - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_scrape"] - regex = "true" - action = "keep" - } - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_job"] - action = "replace" - target_label = "job" - } - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_instance"] - action = "replace" - target_label = "instance" - } - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_path"] - action = "replace" - target_label = "__metrics_path__" - } - - // Choose the pod port - // The discovery generates a target for each declared container port of the pod. - // If the metricsPortName annotation has value, keep only the target where the port name matches the one of the annotation. - rule { - source_labels = ["__meta_kubernetes_pod_container_port_name"] - target_label = "__tmp_port" - } - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_portName"] - regex = "(.+)" - target_label = "__tmp_port" - } - rule { - source_labels = ["__meta_kubernetes_pod_container_port_name"] - action = "keepequal" - target_label = "__tmp_port" - } - - // If the metrics port number annotation has a value, override the target address to use it, regardless whether it is - // one of the declared ports on that Pod. - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_portNumber", "__meta_kubernetes_pod_ip"] - regex = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})" - replacement = "[$2]:$1" // IPv6 - target_label = "__address__" - } - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_portNumber", "__meta_kubernetes_pod_ip"] - regex = "(\\d+);((([0-9]+?)(\\.|$)){4})" // IPv4, takes priority over IPv6 when both exists - replacement = "$2:$1" - target_label = "__address__" - } - - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_scheme"] - action = "replace" - target_label = "__scheme__" - } - - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_scrapeInterval"] - action = "replace" - target_label = "__scrape_interval__" - } - } - discovery.kubernetes "services" { - role = "service" - } - - discovery.relabel "annotation_autodiscovery_services" { - targets = discovery.kubernetes.services.targets - rule { - source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_scrape"] - regex = "true" - action = "keep" - } - rule { - source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_job"] - action = "replace" - target_label = "job" - } - rule { - source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_instance"] - action = "replace" - target_label = "instance" - } - rule { - source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_path"] - action = "replace" - target_label = "__metrics_path__" - } - - // Choose the service port - rule { - source_labels = ["__meta_kubernetes_service_port_name"] - target_label = "__tmp_port" - } - rule { - source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_portName"] - regex = "(.+)" - target_label = "__tmp_port" - } - rule { - source_labels = ["__meta_kubernetes_service_port_name"] - action = "keepequal" - target_label = "__tmp_port" - } - - rule { - source_labels = ["__meta_kubernetes_service_port_number"] - target_label = "__tmp_port" - } - rule { - source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_portNumber"] - regex = "(.+)" - target_label = "__tmp_port" - } - rule { - source_labels = ["__meta_kubernetes_service_port_number"] - action = "keepequal" - target_label = "__tmp_port" - } - - rule { - source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_scheme"] - action = "replace" - target_label = "__scheme__" - } - - rule { - source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_scrapeInterval"] - action = "replace" - target_label = "__scrape_interval__" - } - } - - discovery.relabel "annotation_autodiscovery_http" { - targets = concat(discovery.relabel.annotation_autodiscovery_pods.output, discovery.relabel.annotation_autodiscovery_services.output) - rule { - source_labels = ["__scheme__"] - regex = "https" - action = "drop" - } +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" } +} - discovery.relabel "annotation_autodiscovery_https" { - targets = concat(discovery.relabel.annotation_autodiscovery_pods.output, discovery.relabel.annotation_autodiscovery_services.output) - rule { - source_labels = ["__scheme__"] - regex = "https" - action = "keep" - } +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" } - - prometheus.scrape "annotation_autodiscovery_http" { - targets = discovery.relabel.annotation_autodiscovery_http.output - scrape_interval = "60s" - honor_labels = true - bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" - clustering { - enabled = true - } - - forward_to = [prometheus.relabel.annotation_autodiscovery.receiver] - } - - prometheus.scrape "annotation_autodiscovery_https" { - targets = discovery.relabel.annotation_autodiscovery_https.output - scrape_interval = "60s" - honor_labels = true - bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" - tls_config { - insecure_skip_verify = true - } - clustering { - enabled = true - } - - forward_to = [prometheus.relabel.annotation_autodiscovery.receiver] + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" } +} - prometheus.relabel "annotation_autodiscovery" { - max_cache_size = 100000 - rule { - source_labels = ["__name__"] - regex = "go_*" - action = "drop" - } - forward_to = argument.metrics_destinations.value +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true } -} -annotation_autodiscovery "feature" { - metrics_destinations = [ + forward_to = [ prometheus.remote_write.prometheus.receiver, ] } diff --git a/charts/k8s-monitoring/docs/examples/metrics-tuning/output.yaml b/charts/k8s-monitoring/docs/examples/metrics-tuning/output.yaml index 221b887c89..005d2d1e0f 100644 --- a/charts/k8s-monitoring/docs/examples/metrics-tuning/output.yaml +++ b/charts/k8s-monitoring/docs/examples/metrics-tuning/output.yaml @@ -88,7 +88,7 @@ data: service: services-where: "Name='containerd' or Name='kubelet'" --- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -135,6 +135,218 @@ data: } } } + + // Feature: Annotation Autodiscovery + declare "annotation_autodiscovery" { + argument "metrics_destinations" { + comment = "Must be a list of metric destinations where collected metrics should be forwarded to" + } + + discovery.kubernetes "pods" { + role = "pod" + } + + discovery.relabel "annotation_autodiscovery_pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_scrape"] + regex = "true" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_job"] + action = "replace" + target_label = "job" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_instance"] + action = "replace" + target_label = "instance" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_path"] + action = "replace" + target_label = "__metrics_path__" + } + + // Choose the pod port + // The discovery generates a target for each declared container port of the pod. + // If the metricsPortName annotation has value, keep only the target where the port name matches the one of the annotation. + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_portName"] + regex = "(.+)" + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + action = "keepequal" + target_label = "__tmp_port" + } + + // If the metrics port number annotation has a value, override the target address to use it, regardless whether it is + // one of the declared ports on that Pod. + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_portNumber", "__meta_kubernetes_pod_ip"] + regex = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})" + replacement = "[$2]:$1" // IPv6 + target_label = "__address__" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_portNumber", "__meta_kubernetes_pod_ip"] + regex = "(\\d+);((([0-9]+?)(\\.|$)){4})" // IPv4, takes priority over IPv6 when both exists + replacement = "$2:$1" + target_label = "__address__" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_scheme"] + action = "replace" + target_label = "__scheme__" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_scrapeInterval"] + action = "replace" + target_label = "__scrape_interval__" + } + } + + discovery.kubernetes "services" { + role = "service" + } + + discovery.relabel "annotation_autodiscovery_services" { + targets = discovery.kubernetes.services.targets + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_scrape"] + regex = "true" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_job"] + action = "replace" + target_label = "job" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_instance"] + action = "replace" + target_label = "instance" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_path"] + action = "replace" + target_label = "__metrics_path__" + } + + // Choose the service port + rule { + source_labels = ["__meta_kubernetes_service_port_name"] + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_portName"] + regex = "(.+)" + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_service_port_name"] + action = "keepequal" + target_label = "__tmp_port" + } + + rule { + source_labels = ["__meta_kubernetes_service_port_number"] + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_portNumber"] + regex = "(.+)" + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_service_port_number"] + action = "keepequal" + target_label = "__tmp_port" + } + + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_scheme"] + action = "replace" + target_label = "__scheme__" + } + + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_scrapeInterval"] + action = "replace" + target_label = "__scrape_interval__" + } + } + + discovery.relabel "annotation_autodiscovery_http" { + targets = concat(discovery.relabel.annotation_autodiscovery_pods.output, discovery.relabel.annotation_autodiscovery_services.output) + rule { + source_labels = ["__scheme__"] + regex = "https" + action = "drop" + } + } + + discovery.relabel "annotation_autodiscovery_https" { + targets = concat(discovery.relabel.annotation_autodiscovery_pods.output, discovery.relabel.annotation_autodiscovery_services.output) + rule { + source_labels = ["__scheme__"] + regex = "https" + action = "keep" + } + } + + prometheus.scrape "annotation_autodiscovery_http" { + targets = discovery.relabel.annotation_autodiscovery_http.output + scrape_interval = "60s" + honor_labels = true + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + clustering { + enabled = true + } + + forward_to = [prometheus.relabel.annotation_autodiscovery.receiver] + } + + prometheus.scrape "annotation_autodiscovery_https" { + targets = discovery.relabel.annotation_autodiscovery_https.output + scrape_interval = "60s" + honor_labels = true + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + tls_config { + insecure_skip_verify = true + } + clustering { + enabled = true + } + + forward_to = [prometheus.relabel.annotation_autodiscovery.receiver] + } + + prometheus.relabel "annotation_autodiscovery" { + max_cache_size = 100000 + rule { + source_labels = ["__name__"] + regex = "go_*" + action = "drop" + } + forward_to = argument.metrics_destinations.value + } + } + annotation_autodiscovery "feature" { + metrics_destinations = [ + prometheus.remote_write.prometheus.receiver, + ] + } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -338,216 +550,50 @@ data: prometheus.remote_write.prometheus.receiver, ] } - // Feature: Annotation Autodiscovery - declare "annotation_autodiscovery" { - argument "metrics_destinations" { - comment = "Must be a list of metric destinations where collected metrics should be forwarded to" - } - - discovery.kubernetes "pods" { - role = "pod" - } - - discovery.relabel "annotation_autodiscovery_pods" { - targets = discovery.kubernetes.pods.targets - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_scrape"] - regex = "true" - action = "keep" - } - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_job"] - action = "replace" - target_label = "job" - } - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_instance"] - action = "replace" - target_label = "instance" - } - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_path"] - action = "replace" - target_label = "__metrics_path__" - } - - // Choose the pod port - // The discovery generates a target for each declared container port of the pod. - // If the metricsPortName annotation has value, keep only the target where the port name matches the one of the annotation. - rule { - source_labels = ["__meta_kubernetes_pod_container_port_name"] - target_label = "__tmp_port" - } - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_portName"] - regex = "(.+)" - target_label = "__tmp_port" - } - rule { - source_labels = ["__meta_kubernetes_pod_container_port_name"] - action = "keepequal" - target_label = "__tmp_port" - } - - // If the metrics port number annotation has a value, override the target address to use it, regardless whether it is - // one of the declared ports on that Pod. - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_portNumber", "__meta_kubernetes_pod_ip"] - regex = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})" - replacement = "[$2]:$1" // IPv6 - target_label = "__address__" - } - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_portNumber", "__meta_kubernetes_pod_ip"] - regex = "(\\d+);((([0-9]+?)(\\.|$)){4})" // IPv4, takes priority over IPv6 when both exists - replacement = "$2:$1" - target_label = "__address__" - } - - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_scheme"] - action = "replace" - target_label = "__scheme__" - } - - rule { - source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_scrapeInterval"] - action = "replace" - target_label = "__scrape_interval__" - } - } - - discovery.kubernetes "services" { - role = "service" - } - - discovery.relabel "annotation_autodiscovery_services" { - targets = discovery.kubernetes.services.targets - rule { - source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_scrape"] - regex = "true" - action = "keep" - } - rule { - source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_job"] - action = "replace" - target_label = "job" - } - rule { - source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_instance"] - action = "replace" - target_label = "instance" - } - rule { - source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_path"] - action = "replace" - target_label = "__metrics_path__" - } - - // Choose the service port - rule { - source_labels = ["__meta_kubernetes_service_port_name"] - target_label = "__tmp_port" - } - rule { - source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_portName"] - regex = "(.+)" - target_label = "__tmp_port" - } - rule { - source_labels = ["__meta_kubernetes_service_port_name"] - action = "keepequal" - target_label = "__tmp_port" - } - - rule { - source_labels = ["__meta_kubernetes_service_port_number"] - target_label = "__tmp_port" - } - rule { - source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_portNumber"] - regex = "(.+)" - target_label = "__tmp_port" - } - rule { - source_labels = ["__meta_kubernetes_service_port_number"] - action = "keepequal" - target_label = "__tmp_port" - } - - rule { - source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_scheme"] - action = "replace" - target_label = "__scheme__" - } - - rule { - source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_scrapeInterval"] - action = "replace" - target_label = "__scrape_interval__" - } - } - - discovery.relabel "annotation_autodiscovery_http" { - targets = concat(discovery.relabel.annotation_autodiscovery_pods.output, discovery.relabel.annotation_autodiscovery_services.output) - rule { - source_labels = ["__scheme__"] - regex = "https" - action = "drop" - } - } - discovery.relabel "annotation_autodiscovery_https" { - targets = concat(discovery.relabel.annotation_autodiscovery_pods.output, discovery.relabel.annotation_autodiscovery_services.output) - rule { - source_labels = ["__scheme__"] - regex = "https" - action = "keep" - } + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" } + } - prometheus.scrape "annotation_autodiscovery_http" { - targets = discovery.relabel.annotation_autodiscovery_http.output - scrape_interval = "60s" - honor_labels = true - bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" - clustering { - enabled = true - } - - forward_to = [prometheus.relabel.annotation_autodiscovery.receiver] + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" } - - prometheus.scrape "annotation_autodiscovery_https" { - targets = discovery.relabel.annotation_autodiscovery_https.output - scrape_interval = "60s" - honor_labels = true - bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" - tls_config { - insecure_skip_verify = true - } - clustering { - enabled = true - } - - forward_to = [prometheus.relabel.annotation_autodiscovery.receiver] + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" } + } - prometheus.relabel "annotation_autodiscovery" { - max_cache_size = 100000 - rule { - source_labels = ["__name__"] - regex = "go_*" - action = "drop" - } - forward_to = argument.metrics_destinations.value + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true } - } - annotation_autodiscovery "feature" { - metrics_destinations = [ + forward_to = [ prometheus.remote_write.prometheus.receiver, ] } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{feature="annotationAutodiscovery", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{deployments="kube-state-metrics,node-exporter,windows-exporter", feature="clusterMetrics", sources="kubelet,kubeletResource,cadvisor,kube-state-metrics,node-exporter,windows-exporter", version="1.0.0"} 1 --- # Source: k8s-monitoring/charts/alloy-metrics/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/platforms/azure-aks/alloy-logs.alloy b/charts/k8s-monitoring/docs/examples/platforms/azure-aks/alloy-logs.alloy index adc0793b9e..accca752cc 100644 --- a/charts/k8s-monitoring/docs/examples/platforms/azure-aks/alloy-logs.alloy +++ b/charts/k8s-monitoring/docs/examples/platforms/azure-aks/alloy-logs.alloy @@ -12,6 +12,7 @@ loki.write "loki" { "k8s_cluster_name" = "azure-aks-example-cluster", } } + // Feature: Pod Logs declare "pod_logs" { argument "logs_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/platforms/azure-aks/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/platforms/azure-aks/alloy-metrics.alloy index bbf1c75844..8abfb209a3 100644 --- a/charts/k8s-monitoring/docs/examples/platforms/azure-aks/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/platforms/azure-aks/alloy-metrics.alloy @@ -37,6 +37,7 @@ prometheus.remote_write "prometheus" { } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/platforms/azure-aks/alloy-singleton.alloy b/charts/k8s-monitoring/docs/examples/platforms/azure-aks/alloy-singleton.alloy index 409b6a174b..f7defdc037 100644 --- a/charts/k8s-monitoring/docs/examples/platforms/azure-aks/alloy-singleton.alloy +++ b/charts/k8s-monitoring/docs/examples/platforms/azure-aks/alloy-singleton.alloy @@ -1,3 +1,42 @@ +// Destination: prometheus (prometheus) +otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] +} + +prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + } + tls_config { + insecure_skip_verify = false + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "azure-aks-example-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "azure-aks-example-cluster" + target_label = "cluster" + } + } +} // Destination: loki (loki) otelcol.exporter.loki "loki" { forward_to = [loki.write.loki.receiver] @@ -30,3 +69,37 @@ cluster_events "feature" { loki.write.loki.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/platforms/azure-aks/output.yaml b/charts/k8s-monitoring/docs/examples/platforms/azure-aks/output.yaml index 9562df5833..b66db894f1 100644 --- a/charts/k8s-monitoring/docs/examples/platforms/azure-aks/output.yaml +++ b/charts/k8s-monitoring/docs/examples/platforms/azure-aks/output.yaml @@ -120,145 +120,7 @@ data: service: services-where: "Name='containerd' or Name='kubelet'" --- -# Source: k8s-monitoring/templates/alloy-logs-config.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: ko-alloy-logs - namespace: default -data: - config.alloy: |- - // Destination: loki (loki) - otelcol.exporter.loki "loki" { - forward_to = [loki.write.loki.receiver] - } - - loki.write "loki" { - endpoint { - url = "http://loki.loki.svc:3100/loki/api/v1/push" - } - external_labels = { - cluster = "azure-aks-example-cluster", - "k8s_cluster_name" = "azure-aks-example-cluster", - } - } - // Feature: Pod Logs - declare "pod_logs" { - argument "logs_destinations" { - comment = "Must be a list of log destinations where collected logs should be forwarded to" - } - - discovery.relabel "filtered_pods" { - targets = discovery.kubernetes.pods.targets - rule { - source_labels = ["__meta_kubernetes_namespace"] - action = "replace" - target_label = "namespace" - } - rule { - source_labels = ["__meta_kubernetes_pod_name"] - action = "replace" - target_label = "pod" - } - rule { - source_labels = ["__meta_kubernetes_pod_container_name"] - action = "replace" - target_label = "container" - } - rule { - source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] - separator = "/" - action = "replace" - replacement = "$1" - target_label = "job" - } - - // set the container runtime as a label - rule { - action = "replace" - source_labels = ["__meta_kubernetes_pod_container_id"] - regex = "^(\\S+):\\/\\/.+$" - replacement = "$1" - target_label = "tmp_container_runtime" - } - } - - discovery.kubernetes "pods" { - role = "pod" - selectors { - role = "pod" - field = "spec.nodeName=" + env("HOSTNAME") - } - } - - discovery.relabel "filtered_pods_with_paths" { - targets = discovery.relabel.filtered_pods.output - - rule { - source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] - separator = "/" - action = "replace" - replacement = "/var/log/pods/*$1/*.log" - target_label = "__path__" - } - } - - local.file_match "pod_logs" { - path_targets = discovery.relabel.filtered_pods_with_paths.output - } - - loki.source.file "pod_logs" { - targets = local.file_match.pod_logs.targets - forward_to = [loki.process.pod_logs.receiver] - } - - loki.process "pod_logs" { - stage.match { - selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" - // the cri processing stage extracts the following k/v pairs: log, stream, time, flags - stage.cri {} - - // Set the extract flags and stream values as labels - stage.labels { - values = { - flags = "", - stream = "", - } - } - } - - stage.match { - selector = "{tmp_container_runtime=\"docker\"}" - // the docker processing stage extracts the following k/v pairs: log, stream, time - stage.docker {} - - // Set the extract stream value as a label - stage.labels { - values = { - stream = "", - } - } - } - - // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, - // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary - // container runtime label as it is no longer needed. - stage.label_drop { - values = [ - "filename", - "tmp_container_runtime", - ] - } - forward_to = argument.logs_destinations.value - } - } - pod_logs "feature" { - logs_destinations = [ - loki.write.loki.receiver, - ] - } ---- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -305,6 +167,7 @@ data: } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -509,7 +372,7 @@ data: ] } --- -# Source: k8s-monitoring/templates/alloy-singleton-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -517,6 +380,45 @@ metadata: namespace: default data: config.alloy: |- + // Destination: prometheus (prometheus) + otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] + } + + prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + } + tls_config { + insecure_skip_verify = false + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "azure-aks-example-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "azure-aks-example-cluster" + target_label = "cluster" + } + } + } // Destination: loki (loki) otelcol.exporter.loki "loki" { forward_to = [loki.write.loki.receiver] @@ -549,6 +451,190 @@ data: loki.write.loki.receiver, ] } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{deployments="kube-state-metrics,node-exporter,windows-exporter", feature="clusterMetrics", sources="kubelet,kubeletResource,cadvisor,kube-state-metrics,node-exporter,windows-exporter", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="clusterEvents", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="podLogs", method="volumes", version="1.0.0"} 1 +--- +# Source: k8s-monitoring/templates/alloy-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: ko-alloy-logs + namespace: default +data: + config.alloy: |- + // Destination: loki (loki) + otelcol.exporter.loki "loki" { + forward_to = [loki.write.loki.receiver] + } + + loki.write "loki" { + endpoint { + url = "http://loki.loki.svc:3100/loki/api/v1/push" + } + external_labels = { + cluster = "azure-aks-example-cluster", + "k8s_cluster_name" = "azure-aks-example-cluster", + } + } + + // Feature: Pod Logs + declare "pod_logs" { + argument "logs_destinations" { + comment = "Must be a list of log destinations where collected logs should be forwarded to" + } + + discovery.relabel "filtered_pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "$1" + target_label = "job" + } + + // set the container runtime as a label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_id"] + regex = "^(\\S+):\\/\\/.+$" + replacement = "$1" + target_label = "tmp_container_runtime" + } + } + + discovery.kubernetes "pods" { + role = "pod" + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } + } + + discovery.relabel "filtered_pods_with_paths" { + targets = discovery.relabel.filtered_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "/var/log/pods/*$1/*.log" + target_label = "__path__" + } + } + + local.file_match "pod_logs" { + path_targets = discovery.relabel.filtered_pods_with_paths.output + } + + loki.source.file "pod_logs" { + targets = local.file_match.pod_logs.targets + forward_to = [loki.process.pod_logs.receiver] + } + + loki.process "pod_logs" { + stage.match { + selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" + // the cri processing stage extracts the following k/v pairs: log, stream, time, flags + stage.cri {} + + // Set the extract flags and stream values as labels + stage.labels { + values = { + flags = "", + stream = "", + } + } + } + + stage.match { + selector = "{tmp_container_runtime=\"docker\"}" + // the docker processing stage extracts the following k/v pairs: log, stream, time + stage.docker {} + + // Set the extract stream value as a label + stage.labels { + values = { + stream = "", + } + } + } + + // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, + // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary + // container runtime label as it is no longer needed. + stage.label_drop { + values = [ + "filename", + "tmp_container_runtime", + ] + } + forward_to = argument.logs_destinations.value + } + } + pod_logs "feature" { + logs_destinations = [ + loki.write.loki.receiver, + ] + } --- # Source: k8s-monitoring/charts/alloy-logs/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/platforms/eks-fargate/alloy-logs.alloy b/charts/k8s-monitoring/docs/examples/platforms/eks-fargate/alloy-logs.alloy index ec220f2874..4716b0f543 100644 --- a/charts/k8s-monitoring/docs/examples/platforms/eks-fargate/alloy-logs.alloy +++ b/charts/k8s-monitoring/docs/examples/platforms/eks-fargate/alloy-logs.alloy @@ -12,6 +12,7 @@ loki.write "loki" { "k8s_cluster_name" = "eks-fargate-example-cluster", } } + // Feature: Pod Logs declare "pod_logs" { argument "logs_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/platforms/eks-fargate/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/platforms/eks-fargate/alloy-metrics.alloy index 44facb8ce2..deaf0b1c67 100644 --- a/charts/k8s-monitoring/docs/examples/platforms/eks-fargate/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/platforms/eks-fargate/alloy-metrics.alloy @@ -37,6 +37,7 @@ prometheus.remote_write "prometheus" { } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/platforms/eks-fargate/alloy-singleton.alloy b/charts/k8s-monitoring/docs/examples/platforms/eks-fargate/alloy-singleton.alloy index 13ed0378ff..6d302f6ac1 100644 --- a/charts/k8s-monitoring/docs/examples/platforms/eks-fargate/alloy-singleton.alloy +++ b/charts/k8s-monitoring/docs/examples/platforms/eks-fargate/alloy-singleton.alloy @@ -1,3 +1,42 @@ +// Destination: prometheus (prometheus) +otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] +} + +prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + } + tls_config { + insecure_skip_verify = false + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "eks-fargate-example-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "eks-fargate-example-cluster" + target_label = "cluster" + } + } +} // Destination: loki (loki) otelcol.exporter.loki "loki" { forward_to = [loki.write.loki.receiver] @@ -30,3 +69,37 @@ cluster_events "feature" { loki.write.loki.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/platforms/eks-fargate/output.yaml b/charts/k8s-monitoring/docs/examples/platforms/eks-fargate/output.yaml index 3b42ebdb9a..642e2e7ecd 100644 --- a/charts/k8s-monitoring/docs/examples/platforms/eks-fargate/output.yaml +++ b/charts/k8s-monitoring/docs/examples/platforms/eks-fargate/output.yaml @@ -103,125 +103,7 @@ data: service: services-where: "Name='containerd' or Name='kubelet'" --- -# Source: k8s-monitoring/templates/alloy-logs-config.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: ko-alloy-logs - namespace: default -data: - config.alloy: |- - // Destination: loki (loki) - otelcol.exporter.loki "loki" { - forward_to = [loki.write.loki.receiver] - } - - loki.write "loki" { - endpoint { - url = "http://loki.loki.svc:3100/loki/api/v1/push" - } - external_labels = { - cluster = "eks-fargate-example-cluster", - "k8s_cluster_name" = "eks-fargate-example-cluster", - } - } - // Feature: Pod Logs - declare "pod_logs" { - argument "logs_destinations" { - comment = "Must be a list of log destinations where collected logs should be forwarded to" - } - - discovery.relabel "filtered_pods" { - targets = discovery.kubernetes.pods.targets - rule { - source_labels = ["__meta_kubernetes_namespace"] - action = "replace" - target_label = "namespace" - } - rule { - source_labels = ["__meta_kubernetes_pod_name"] - action = "replace" - target_label = "pod" - } - rule { - source_labels = ["__meta_kubernetes_pod_container_name"] - action = "replace" - target_label = "container" - } - rule { - source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] - separator = "/" - action = "replace" - replacement = "$1" - target_label = "job" - } - - // set the container runtime as a label - rule { - action = "replace" - source_labels = ["__meta_kubernetes_pod_container_id"] - regex = "^(\\S+):\\/\\/.+$" - replacement = "$1" - target_label = "tmp_container_runtime" - } - } - - discovery.kubernetes "pods" { - role = "pod" - } - - loki.source.kubernetes "pod_logs" { - targets = discovery.relabel.filtered_pods.output - forward_to = [loki.process.pod_logs.receiver] - } - - loki.process "pod_logs" { - stage.match { - selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" - // the cri processing stage extracts the following k/v pairs: log, stream, time, flags - stage.cri {} - - // Set the extract flags and stream values as labels - stage.labels { - values = { - flags = "", - stream = "", - } - } - } - - stage.match { - selector = "{tmp_container_runtime=\"docker\"}" - // the docker processing stage extracts the following k/v pairs: log, stream, time - stage.docker {} - - // Set the extract stream value as a label - stage.labels { - values = { - stream = "", - } - } - } - - // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, - // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary - // container runtime label as it is no longer needed. - stage.label_drop { - values = [ - "filename", - "tmp_container_runtime", - ] - } - forward_to = argument.logs_destinations.value - } - } - pod_logs "feature" { - logs_destinations = [ - loki.write.loki.receiver, - ] - } ---- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -268,6 +150,7 @@ data: } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -439,7 +322,7 @@ data: ] } --- -# Source: k8s-monitoring/templates/alloy-singleton-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -447,6 +330,45 @@ metadata: namespace: default data: config.alloy: |- + // Destination: prometheus (prometheus) + otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] + } + + prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + } + tls_config { + insecure_skip_verify = false + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "eks-fargate-example-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "eks-fargate-example-cluster" + target_label = "cluster" + } + } + } // Destination: loki (loki) otelcol.exporter.loki "loki" { forward_to = [loki.write.loki.receiver] @@ -479,6 +401,170 @@ data: loki.write.loki.receiver, ] } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{deployments="kube-state-metrics,windows-exporter", feature="clusterMetrics", sources="kubelet,kubeletResource,cadvisor,kube-state-metrics,windows-exporter", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="clusterEvents", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="podLogs", method="kubernetesApi", version="1.0.0"} 1 +--- +# Source: k8s-monitoring/templates/alloy-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: ko-alloy-logs + namespace: default +data: + config.alloy: |- + // Destination: loki (loki) + otelcol.exporter.loki "loki" { + forward_to = [loki.write.loki.receiver] + } + + loki.write "loki" { + endpoint { + url = "http://loki.loki.svc:3100/loki/api/v1/push" + } + external_labels = { + cluster = "eks-fargate-example-cluster", + "k8s_cluster_name" = "eks-fargate-example-cluster", + } + } + + // Feature: Pod Logs + declare "pod_logs" { + argument "logs_destinations" { + comment = "Must be a list of log destinations where collected logs should be forwarded to" + } + + discovery.relabel "filtered_pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "$1" + target_label = "job" + } + + // set the container runtime as a label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_id"] + regex = "^(\\S+):\\/\\/.+$" + replacement = "$1" + target_label = "tmp_container_runtime" + } + } + + discovery.kubernetes "pods" { + role = "pod" + } + + loki.source.kubernetes "pod_logs" { + targets = discovery.relabel.filtered_pods.output + forward_to = [loki.process.pod_logs.receiver] + } + + loki.process "pod_logs" { + stage.match { + selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" + // the cri processing stage extracts the following k/v pairs: log, stream, time, flags + stage.cri {} + + // Set the extract flags and stream values as labels + stage.labels { + values = { + flags = "", + stream = "", + } + } + } + + stage.match { + selector = "{tmp_container_runtime=\"docker\"}" + // the docker processing stage extracts the following k/v pairs: log, stream, time + stage.docker {} + + // Set the extract stream value as a label + stage.labels { + values = { + stream = "", + } + } + } + + // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, + // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary + // container runtime label as it is no longer needed. + stage.label_drop { + values = [ + "filename", + "tmp_container_runtime", + ] + } + forward_to = argument.logs_destinations.value + } + } + pod_logs "feature" { + logs_destinations = [ + loki.write.loki.receiver, + ] + } --- # Source: k8s-monitoring/charts/alloy-logs/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/platforms/gke-autopilot/alloy-logs.alloy b/charts/k8s-monitoring/docs/examples/platforms/gke-autopilot/alloy-logs.alloy index 71df3a3922..39323c0d7e 100644 --- a/charts/k8s-monitoring/docs/examples/platforms/gke-autopilot/alloy-logs.alloy +++ b/charts/k8s-monitoring/docs/examples/platforms/gke-autopilot/alloy-logs.alloy @@ -12,6 +12,7 @@ loki.write "loki" { "k8s_cluster_name" = "gke-autopilot-example-cluster", } } + // Feature: Pod Logs declare "pod_logs" { argument "logs_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/platforms/gke-autopilot/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/platforms/gke-autopilot/alloy-metrics.alloy index f86c9040dd..608776a9a3 100644 --- a/charts/k8s-monitoring/docs/examples/platforms/gke-autopilot/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/platforms/gke-autopilot/alloy-metrics.alloy @@ -37,6 +37,7 @@ prometheus.remote_write "prometheus" { } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/platforms/gke-autopilot/alloy-singleton.alloy b/charts/k8s-monitoring/docs/examples/platforms/gke-autopilot/alloy-singleton.alloy index 6f5690af3e..30aa817496 100644 --- a/charts/k8s-monitoring/docs/examples/platforms/gke-autopilot/alloy-singleton.alloy +++ b/charts/k8s-monitoring/docs/examples/platforms/gke-autopilot/alloy-singleton.alloy @@ -1,3 +1,42 @@ +// Destination: prometheus (prometheus) +otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] +} + +prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + } + tls_config { + insecure_skip_verify = false + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "gke-autopilot-example-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "gke-autopilot-example-cluster" + target_label = "cluster" + } + } +} // Destination: loki (loki) otelcol.exporter.loki "loki" { forward_to = [loki.write.loki.receiver] @@ -30,3 +69,37 @@ cluster_events "feature" { loki.write.loki.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/platforms/gke-autopilot/output.yaml b/charts/k8s-monitoring/docs/examples/platforms/gke-autopilot/output.yaml index 67d28e64d8..6d31ff0a68 100644 --- a/charts/k8s-monitoring/docs/examples/platforms/gke-autopilot/output.yaml +++ b/charts/k8s-monitoring/docs/examples/platforms/gke-autopilot/output.yaml @@ -103,145 +103,7 @@ data: service: services-where: "Name='containerd' or Name='kubelet'" --- -# Source: k8s-monitoring/templates/alloy-logs-config.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: ko-alloy-logs - namespace: default -data: - config.alloy: |- - // Destination: loki (loki) - otelcol.exporter.loki "loki" { - forward_to = [loki.write.loki.receiver] - } - - loki.write "loki" { - endpoint { - url = "http://loki.loki.svc:3100/loki/api/v1/push" - } - external_labels = { - cluster = "gke-autopilot-example-cluster", - "k8s_cluster_name" = "gke-autopilot-example-cluster", - } - } - // Feature: Pod Logs - declare "pod_logs" { - argument "logs_destinations" { - comment = "Must be a list of log destinations where collected logs should be forwarded to" - } - - discovery.relabel "filtered_pods" { - targets = discovery.kubernetes.pods.targets - rule { - source_labels = ["__meta_kubernetes_namespace"] - action = "replace" - target_label = "namespace" - } - rule { - source_labels = ["__meta_kubernetes_pod_name"] - action = "replace" - target_label = "pod" - } - rule { - source_labels = ["__meta_kubernetes_pod_container_name"] - action = "replace" - target_label = "container" - } - rule { - source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] - separator = "/" - action = "replace" - replacement = "$1" - target_label = "job" - } - - // set the container runtime as a label - rule { - action = "replace" - source_labels = ["__meta_kubernetes_pod_container_id"] - regex = "^(\\S+):\\/\\/.+$" - replacement = "$1" - target_label = "tmp_container_runtime" - } - } - - discovery.kubernetes "pods" { - role = "pod" - selectors { - role = "pod" - field = "spec.nodeName=" + env("HOSTNAME") - } - } - - discovery.relabel "filtered_pods_with_paths" { - targets = discovery.relabel.filtered_pods.output - - rule { - source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] - separator = "/" - action = "replace" - replacement = "/var/log/pods/*$1/*.log" - target_label = "__path__" - } - } - - local.file_match "pod_logs" { - path_targets = discovery.relabel.filtered_pods_with_paths.output - } - - loki.source.file "pod_logs" { - targets = local.file_match.pod_logs.targets - forward_to = [loki.process.pod_logs.receiver] - } - - loki.process "pod_logs" { - stage.match { - selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" - // the cri processing stage extracts the following k/v pairs: log, stream, time, flags - stage.cri {} - - // Set the extract flags and stream values as labels - stage.labels { - values = { - flags = "", - stream = "", - } - } - } - - stage.match { - selector = "{tmp_container_runtime=\"docker\"}" - // the docker processing stage extracts the following k/v pairs: log, stream, time - stage.docker {} - - // Set the extract stream value as a label - stage.labels { - values = { - stream = "", - } - } - } - - // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, - // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary - // container runtime label as it is no longer needed. - stage.label_drop { - values = [ - "filename", - "tmp_container_runtime", - ] - } - forward_to = argument.logs_destinations.value - } - } - pod_logs "feature" { - logs_destinations = [ - loki.write.loki.receiver, - ] - } ---- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -288,6 +150,7 @@ data: } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -459,7 +322,7 @@ data: ] } --- -# Source: k8s-monitoring/templates/alloy-singleton-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -467,6 +330,45 @@ metadata: namespace: default data: config.alloy: |- + // Destination: prometheus (prometheus) + otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] + } + + prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + } + tls_config { + insecure_skip_verify = false + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "gke-autopilot-example-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "gke-autopilot-example-cluster" + target_label = "cluster" + } + } + } // Destination: loki (loki) otelcol.exporter.loki "loki" { forward_to = [loki.write.loki.receiver] @@ -499,6 +401,190 @@ data: loki.write.loki.receiver, ] } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{deployments="kube-state-metrics,windows-exporter", feature="clusterMetrics", sources="kubelet,kubeletResource,cadvisor,kube-state-metrics,windows-exporter", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="clusterEvents", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="podLogs", method="volumes", version="1.0.0"} 1 +--- +# Source: k8s-monitoring/templates/alloy-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: ko-alloy-logs + namespace: default +data: + config.alloy: |- + // Destination: loki (loki) + otelcol.exporter.loki "loki" { + forward_to = [loki.write.loki.receiver] + } + + loki.write "loki" { + endpoint { + url = "http://loki.loki.svc:3100/loki/api/v1/push" + } + external_labels = { + cluster = "gke-autopilot-example-cluster", + "k8s_cluster_name" = "gke-autopilot-example-cluster", + } + } + + // Feature: Pod Logs + declare "pod_logs" { + argument "logs_destinations" { + comment = "Must be a list of log destinations where collected logs should be forwarded to" + } + + discovery.relabel "filtered_pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "$1" + target_label = "job" + } + + // set the container runtime as a label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_id"] + regex = "^(\\S+):\\/\\/.+$" + replacement = "$1" + target_label = "tmp_container_runtime" + } + } + + discovery.kubernetes "pods" { + role = "pod" + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } + } + + discovery.relabel "filtered_pods_with_paths" { + targets = discovery.relabel.filtered_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "/var/log/pods/*$1/*.log" + target_label = "__path__" + } + } + + local.file_match "pod_logs" { + path_targets = discovery.relabel.filtered_pods_with_paths.output + } + + loki.source.file "pod_logs" { + targets = local.file_match.pod_logs.targets + forward_to = [loki.process.pod_logs.receiver] + } + + loki.process "pod_logs" { + stage.match { + selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" + // the cri processing stage extracts the following k/v pairs: log, stream, time, flags + stage.cri {} + + // Set the extract flags and stream values as labels + stage.labels { + values = { + flags = "", + stream = "", + } + } + } + + stage.match { + selector = "{tmp_container_runtime=\"docker\"}" + // the docker processing stage extracts the following k/v pairs: log, stream, time + stage.docker {} + + // Set the extract stream value as a label + stage.labels { + values = { + stream = "", + } + } + } + + // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, + // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary + // container runtime label as it is no longer needed. + stage.label_drop { + values = [ + "filename", + "tmp_container_runtime", + ] + } + forward_to = argument.logs_destinations.value + } + } + pod_logs "feature" { + logs_destinations = [ + loki.write.loki.receiver, + ] + } --- # Source: k8s-monitoring/charts/alloy-logs/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/platforms/openshift/alloy-logs.alloy b/charts/k8s-monitoring/docs/examples/platforms/openshift/alloy-logs.alloy index 35d16a17d6..4640f9483e 100644 --- a/charts/k8s-monitoring/docs/examples/platforms/openshift/alloy-logs.alloy +++ b/charts/k8s-monitoring/docs/examples/platforms/openshift/alloy-logs.alloy @@ -12,6 +12,7 @@ loki.write "loki" { "k8s_cluster_name" = "openshift-cluster", } } + // Feature: Pod Logs declare "pod_logs" { argument "logs_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/platforms/openshift/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/platforms/openshift/alloy-metrics.alloy index b6b161fc50..3dbe12e0fe 100644 --- a/charts/k8s-monitoring/docs/examples/platforms/openshift/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/platforms/openshift/alloy-metrics.alloy @@ -37,6 +37,7 @@ prometheus.remote_write "prometheus" { } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/platforms/openshift/alloy-singleton.alloy b/charts/k8s-monitoring/docs/examples/platforms/openshift/alloy-singleton.alloy index 122c63e71b..331af4d6d4 100644 --- a/charts/k8s-monitoring/docs/examples/platforms/openshift/alloy-singleton.alloy +++ b/charts/k8s-monitoring/docs/examples/platforms/openshift/alloy-singleton.alloy @@ -1,3 +1,42 @@ +// Destination: prometheus (prometheus) +otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] +} + +prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + } + tls_config { + insecure_skip_verify = false + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "openshift-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "openshift-cluster" + target_label = "cluster" + } + } +} // Destination: loki (loki) otelcol.exporter.loki "loki" { forward_to = [loki.write.loki.receiver] @@ -30,3 +69,37 @@ cluster_events "feature" { loki.write.loki.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/platforms/openshift/output.yaml b/charts/k8s-monitoring/docs/examples/platforms/openshift/output.yaml index 83556df117..eaba684416 100644 --- a/charts/k8s-monitoring/docs/examples/platforms/openshift/output.yaml +++ b/charts/k8s-monitoring/docs/examples/platforms/openshift/output.yaml @@ -133,145 +133,7 @@ data: service: services-where: "Name='containerd' or Name='kubelet'" --- -# Source: k8s-monitoring/templates/alloy-logs-config.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: ko-alloy-logs - namespace: default -data: - config.alloy: |- - // Destination: loki (loki) - otelcol.exporter.loki "loki" { - forward_to = [loki.write.loki.receiver] - } - - loki.write "loki" { - endpoint { - url = "http://prometheus.prometheus.svc:9090/api/v1/write" - } - external_labels = { - cluster = "openshift-cluster", - "k8s_cluster_name" = "openshift-cluster", - } - } - // Feature: Pod Logs - declare "pod_logs" { - argument "logs_destinations" { - comment = "Must be a list of log destinations where collected logs should be forwarded to" - } - - discovery.relabel "filtered_pods" { - targets = discovery.kubernetes.pods.targets - rule { - source_labels = ["__meta_kubernetes_namespace"] - action = "replace" - target_label = "namespace" - } - rule { - source_labels = ["__meta_kubernetes_pod_name"] - action = "replace" - target_label = "pod" - } - rule { - source_labels = ["__meta_kubernetes_pod_container_name"] - action = "replace" - target_label = "container" - } - rule { - source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] - separator = "/" - action = "replace" - replacement = "$1" - target_label = "job" - } - - // set the container runtime as a label - rule { - action = "replace" - source_labels = ["__meta_kubernetes_pod_container_id"] - regex = "^(\\S+):\\/\\/.+$" - replacement = "$1" - target_label = "tmp_container_runtime" - } - } - - discovery.kubernetes "pods" { - role = "pod" - selectors { - role = "pod" - field = "spec.nodeName=" + env("HOSTNAME") - } - } - - discovery.relabel "filtered_pods_with_paths" { - targets = discovery.relabel.filtered_pods.output - - rule { - source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] - separator = "/" - action = "replace" - replacement = "/var/log/pods/*$1/*.log" - target_label = "__path__" - } - } - - local.file_match "pod_logs" { - path_targets = discovery.relabel.filtered_pods_with_paths.output - } - - loki.source.file "pod_logs" { - targets = local.file_match.pod_logs.targets - forward_to = [loki.process.pod_logs.receiver] - } - - loki.process "pod_logs" { - stage.match { - selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" - // the cri processing stage extracts the following k/v pairs: log, stream, time, flags - stage.cri {} - - // Set the extract flags and stream values as labels - stage.labels { - values = { - flags = "", - stream = "", - } - } - } - - stage.match { - selector = "{tmp_container_runtime=\"docker\"}" - // the docker processing stage extracts the following k/v pairs: log, stream, time - stage.docker {} - - // Set the extract stream value as a label - stage.labels { - values = { - stream = "", - } - } - } - - // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, - // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary - // container runtime label as it is no longer needed. - stage.label_drop { - values = [ - "filename", - "tmp_container_runtime", - ] - } - forward_to = argument.logs_destinations.value - } - } - pod_logs "feature" { - logs_destinations = [ - loki.write.loki.receiver, - ] - } ---- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -318,6 +180,7 @@ data: } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -563,7 +426,7 @@ data: ] } --- -# Source: k8s-monitoring/templates/alloy-singleton-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -571,6 +434,45 @@ metadata: namespace: default data: config.alloy: |- + // Destination: prometheus (prometheus) + otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] + } + + prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + } + tls_config { + insecure_skip_verify = false + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "openshift-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "openshift-cluster" + target_label = "cluster" + } + } + } // Destination: loki (loki) otelcol.exporter.loki "loki" { forward_to = [loki.write.loki.receiver] @@ -603,6 +505,190 @@ data: loki.write.loki.receiver, ] } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform="openshift"} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{deployments="kube-state-metrics,node-exporter,windows-exporter,kepler", feature="clusterMetrics", sources="kubelet,kubeletResource,cadvisor,kube-state-metrics,node-exporter,windows-exporter,kepler", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="clusterEvents", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="podLogs", method="volumes", version="1.0.0"} 1 +--- +# Source: k8s-monitoring/templates/alloy-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: ko-alloy-logs + namespace: default +data: + config.alloy: |- + // Destination: loki (loki) + otelcol.exporter.loki "loki" { + forward_to = [loki.write.loki.receiver] + } + + loki.write "loki" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + } + external_labels = { + cluster = "openshift-cluster", + "k8s_cluster_name" = "openshift-cluster", + } + } + + // Feature: Pod Logs + declare "pod_logs" { + argument "logs_destinations" { + comment = "Must be a list of log destinations where collected logs should be forwarded to" + } + + discovery.relabel "filtered_pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "$1" + target_label = "job" + } + + // set the container runtime as a label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_id"] + regex = "^(\\S+):\\/\\/.+$" + replacement = "$1" + target_label = "tmp_container_runtime" + } + } + + discovery.kubernetes "pods" { + role = "pod" + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } + } + + discovery.relabel "filtered_pods_with_paths" { + targets = discovery.relabel.filtered_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "/var/log/pods/*$1/*.log" + target_label = "__path__" + } + } + + local.file_match "pod_logs" { + path_targets = discovery.relabel.filtered_pods_with_paths.output + } + + loki.source.file "pod_logs" { + targets = local.file_match.pod_logs.targets + forward_to = [loki.process.pod_logs.receiver] + } + + loki.process "pod_logs" { + stage.match { + selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" + // the cri processing stage extracts the following k/v pairs: log, stream, time, flags + stage.cri {} + + // Set the extract flags and stream values as labels + stage.labels { + values = { + flags = "", + stream = "", + } + } + } + + stage.match { + selector = "{tmp_container_runtime=\"docker\"}" + // the docker processing stage extracts the following k/v pairs: log, stream, time + stage.docker {} + + // Set the extract stream value as a label + stage.labels { + values = { + stream = "", + } + } + } + + // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, + // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary + // container runtime label as it is no longer needed. + stage.label_drop { + values = [ + "filename", + "tmp_container_runtime", + ] + } + forward_to = argument.logs_destinations.value + } + } + pod_logs "feature" { + logs_destinations = [ + loki.write.loki.receiver, + ] + } --- # Source: k8s-monitoring/charts/alloy-logs/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/private-image-registries/alloy-logs.alloy b/charts/k8s-monitoring/docs/examples/private-image-registries/alloy-logs.alloy index e93d5717ec..3ac836d92d 100644 --- a/charts/k8s-monitoring/docs/examples/private-image-registries/alloy-logs.alloy +++ b/charts/k8s-monitoring/docs/examples/private-image-registries/alloy-logs.alloy @@ -12,6 +12,7 @@ loki.write "loki" { "k8s_cluster_name" = "private-image-registries-example-cluster", } } + // Feature: Pod Logs declare "pod_logs" { argument "logs_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/private-image-registries/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/private-image-registries/alloy-metrics.alloy index 40409053d9..b90e9a4a61 100644 --- a/charts/k8s-monitoring/docs/examples/private-image-registries/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/private-image-registries/alloy-metrics.alloy @@ -37,6 +37,7 @@ prometheus.remote_write "prometheus" { } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -240,3 +241,37 @@ cluster_metrics "feature" { prometheus.remote_write.prometheus.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/private-image-registries/output.yaml b/charts/k8s-monitoring/docs/examples/private-image-registries/output.yaml index 25cf95fc8f..5fe53f9311 100644 --- a/charts/k8s-monitoring/docs/examples/private-image-registries/output.yaml +++ b/charts/k8s-monitoring/docs/examples/private-image-registries/output.yaml @@ -110,145 +110,7 @@ data: service: services-where: "Name='containerd' or Name='kubelet'" --- -# Source: k8s-monitoring/templates/alloy-logs-config.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: ko-alloy-logs - namespace: default -data: - config.alloy: |- - // Destination: loki (loki) - otelcol.exporter.loki "loki" { - forward_to = [loki.write.loki.receiver] - } - - loki.write "loki" { - endpoint { - url = "http://loki.loki.svc:3100/loki/api/v1/push" - } - external_labels = { - cluster = "private-image-registries-example-cluster", - "k8s_cluster_name" = "private-image-registries-example-cluster", - } - } - // Feature: Pod Logs - declare "pod_logs" { - argument "logs_destinations" { - comment = "Must be a list of log destinations where collected logs should be forwarded to" - } - - discovery.relabel "filtered_pods" { - targets = discovery.kubernetes.pods.targets - rule { - source_labels = ["__meta_kubernetes_namespace"] - action = "replace" - target_label = "namespace" - } - rule { - source_labels = ["__meta_kubernetes_pod_name"] - action = "replace" - target_label = "pod" - } - rule { - source_labels = ["__meta_kubernetes_pod_container_name"] - action = "replace" - target_label = "container" - } - rule { - source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] - separator = "/" - action = "replace" - replacement = "$1" - target_label = "job" - } - - // set the container runtime as a label - rule { - action = "replace" - source_labels = ["__meta_kubernetes_pod_container_id"] - regex = "^(\\S+):\\/\\/.+$" - replacement = "$1" - target_label = "tmp_container_runtime" - } - } - - discovery.kubernetes "pods" { - role = "pod" - selectors { - role = "pod" - field = "spec.nodeName=" + env("HOSTNAME") - } - } - - discovery.relabel "filtered_pods_with_paths" { - targets = discovery.relabel.filtered_pods.output - - rule { - source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] - separator = "/" - action = "replace" - replacement = "/var/log/pods/*$1/*.log" - target_label = "__path__" - } - } - - local.file_match "pod_logs" { - path_targets = discovery.relabel.filtered_pods_with_paths.output - } - - loki.source.file "pod_logs" { - targets = local.file_match.pod_logs.targets - forward_to = [loki.process.pod_logs.receiver] - } - - loki.process "pod_logs" { - stage.match { - selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" - // the cri processing stage extracts the following k/v pairs: log, stream, time, flags - stage.cri {} - - // Set the extract flags and stream values as labels - stage.labels { - values = { - flags = "", - stream = "", - } - } - } - - stage.match { - selector = "{tmp_container_runtime=\"docker\"}" - // the docker processing stage extracts the following k/v pairs: log, stream, time - stage.docker {} - - // Set the extract stream value as a label - stage.labels { - values = { - stream = "", - } - } - } - - // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, - // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary - // container runtime label as it is no longer needed. - stage.label_drop { - values = [ - "filename", - "tmp_container_runtime", - ] - } - forward_to = argument.logs_destinations.value - } - } - pod_logs "feature" { - logs_destinations = [ - loki.write.loki.receiver, - ] - } ---- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -295,6 +157,7 @@ data: } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -498,6 +361,189 @@ data: prometheus.remote_write.prometheus.receiver, ] } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{deployments="kube-state-metrics,node-exporter,windows-exporter", feature="clusterMetrics", sources="kubelet,kubeletResource,cadvisor,kube-state-metrics,node-exporter,windows-exporter", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="podLogs", method="volumes", version="1.0.0"} 1 +--- +# Source: k8s-monitoring/templates/alloy-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: ko-alloy-logs + namespace: default +data: + config.alloy: |- + // Destination: loki (loki) + otelcol.exporter.loki "loki" { + forward_to = [loki.write.loki.receiver] + } + + loki.write "loki" { + endpoint { + url = "http://loki.loki.svc:3100/loki/api/v1/push" + } + external_labels = { + cluster = "private-image-registries-example-cluster", + "k8s_cluster_name" = "private-image-registries-example-cluster", + } + } + + // Feature: Pod Logs + declare "pod_logs" { + argument "logs_destinations" { + comment = "Must be a list of log destinations where collected logs should be forwarded to" + } + + discovery.relabel "filtered_pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "$1" + target_label = "job" + } + + // set the container runtime as a label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_id"] + regex = "^(\\S+):\\/\\/.+$" + replacement = "$1" + target_label = "tmp_container_runtime" + } + } + + discovery.kubernetes "pods" { + role = "pod" + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } + } + + discovery.relabel "filtered_pods_with_paths" { + targets = discovery.relabel.filtered_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "/var/log/pods/*$1/*.log" + target_label = "__path__" + } + } + + local.file_match "pod_logs" { + path_targets = discovery.relabel.filtered_pods_with_paths.output + } + + loki.source.file "pod_logs" { + targets = local.file_match.pod_logs.targets + forward_to = [loki.process.pod_logs.receiver] + } + + loki.process "pod_logs" { + stage.match { + selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" + // the cri processing stage extracts the following k/v pairs: log, stream, time, flags + stage.cri {} + + // Set the extract flags and stream values as labels + stage.labels { + values = { + flags = "", + stream = "", + } + } + } + + stage.match { + selector = "{tmp_container_runtime=\"docker\"}" + // the docker processing stage extracts the following k/v pairs: log, stream, time + stage.docker {} + + // Set the extract stream value as a label + stage.labels { + values = { + stream = "", + } + } + } + + // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, + // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary + // container runtime label as it is no longer needed. + stage.label_drop { + values = [ + "filename", + "tmp_container_runtime", + ] + } + forward_to = argument.logs_destinations.value + } + } + pod_logs "feature" { + logs_destinations = [ + loki.write.loki.receiver, + ] + } --- # Source: k8s-monitoring/charts/alloy-logs/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/k8s-monitoring/docs/examples/proxies/alloy-logs.alloy b/charts/k8s-monitoring/docs/examples/proxies/alloy-logs.alloy index d1e02c093c..4245daa883 100644 --- a/charts/k8s-monitoring/docs/examples/proxies/alloy-logs.alloy +++ b/charts/k8s-monitoring/docs/examples/proxies/alloy-logs.alloy @@ -12,6 +12,7 @@ loki.write "loki" { "k8s_cluster_name" = "proxies-example-cluster", } } + // Feature: Pod Logs declare "pod_logs" { argument "logs_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/proxies/alloy-metrics.alloy b/charts/k8s-monitoring/docs/examples/proxies/alloy-metrics.alloy index b459826492..94d7b5a81a 100644 --- a/charts/k8s-monitoring/docs/examples/proxies/alloy-metrics.alloy +++ b/charts/k8s-monitoring/docs/examples/proxies/alloy-metrics.alloy @@ -38,6 +38,7 @@ prometheus.remote_write "prometheus" { } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/proxies/alloy-receiver.alloy b/charts/k8s-monitoring/docs/examples/proxies/alloy-receiver.alloy index e860c7cce0..7d031c5ad4 100644 --- a/charts/k8s-monitoring/docs/examples/proxies/alloy-receiver.alloy +++ b/charts/k8s-monitoring/docs/examples/proxies/alloy-receiver.alloy @@ -87,7 +87,6 @@ otelcol.exporter.otlp "tempo" { } } - // Feature: Application Observability declare "application_observability" { argument "metrics_destinations" { diff --git a/charts/k8s-monitoring/docs/examples/proxies/alloy-singleton.alloy b/charts/k8s-monitoring/docs/examples/proxies/alloy-singleton.alloy index 7dee745264..5d0fe09266 100644 --- a/charts/k8s-monitoring/docs/examples/proxies/alloy-singleton.alloy +++ b/charts/k8s-monitoring/docs/examples/proxies/alloy-singleton.alloy @@ -1,3 +1,43 @@ +// Destination: prometheus (prometheus) +otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] +} + +prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + } + proxy_url = "https://myproxy.default.svc:8080" + tls_config { + insecure_skip_verify = false + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "proxies-example-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "proxies-example-cluster" + target_label = "cluster" + } + } +} // Destination: loki (loki) otelcol.exporter.loki "loki" { forward_to = [loki.write.loki.receiver] @@ -30,3 +70,37 @@ cluster_events "feature" { loki.write.loki.receiver, ] } + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] +} diff --git a/charts/k8s-monitoring/docs/examples/proxies/output.yaml b/charts/k8s-monitoring/docs/examples/proxies/output.yaml index eb86309a3d..ea39d5000d 100644 --- a/charts/k8s-monitoring/docs/examples/proxies/output.yaml +++ b/charts/k8s-monitoring/docs/examples/proxies/output.yaml @@ -152,145 +152,7 @@ data: service: services-where: "Name='containerd' or Name='kubelet'" --- -# Source: k8s-monitoring/templates/alloy-logs-config.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: ko-alloy-logs - namespace: default -data: - config.alloy: |- - // Destination: loki (loki) - otelcol.exporter.loki "loki" { - forward_to = [loki.write.loki.receiver] - } - - loki.write "loki" { - endpoint { - url = "http://loki.loki.svc:3100/loki/api/v1/push" - } - external_labels = { - cluster = "proxies-example-cluster", - "k8s_cluster_name" = "proxies-example-cluster", - } - } - // Feature: Pod Logs - declare "pod_logs" { - argument "logs_destinations" { - comment = "Must be a list of log destinations where collected logs should be forwarded to" - } - - discovery.relabel "filtered_pods" { - targets = discovery.kubernetes.pods.targets - rule { - source_labels = ["__meta_kubernetes_namespace"] - action = "replace" - target_label = "namespace" - } - rule { - source_labels = ["__meta_kubernetes_pod_name"] - action = "replace" - target_label = "pod" - } - rule { - source_labels = ["__meta_kubernetes_pod_container_name"] - action = "replace" - target_label = "container" - } - rule { - source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] - separator = "/" - action = "replace" - replacement = "$1" - target_label = "job" - } - - // set the container runtime as a label - rule { - action = "replace" - source_labels = ["__meta_kubernetes_pod_container_id"] - regex = "^(\\S+):\\/\\/.+$" - replacement = "$1" - target_label = "tmp_container_runtime" - } - } - - discovery.kubernetes "pods" { - role = "pod" - selectors { - role = "pod" - field = "spec.nodeName=" + env("HOSTNAME") - } - } - - discovery.relabel "filtered_pods_with_paths" { - targets = discovery.relabel.filtered_pods.output - - rule { - source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] - separator = "/" - action = "replace" - replacement = "/var/log/pods/*$1/*.log" - target_label = "__path__" - } - } - - local.file_match "pod_logs" { - path_targets = discovery.relabel.filtered_pods_with_paths.output - } - - loki.source.file "pod_logs" { - targets = local.file_match.pod_logs.targets - forward_to = [loki.process.pod_logs.receiver] - } - - loki.process "pod_logs" { - stage.match { - selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" - // the cri processing stage extracts the following k/v pairs: log, stream, time, flags - stage.cri {} - - // Set the extract flags and stream values as labels - stage.labels { - values = { - flags = "", - stream = "", - } - } - } - - stage.match { - selector = "{tmp_container_runtime=\"docker\"}" - // the docker processing stage extracts the following k/v pairs: log, stream, time - stage.docker {} - - // Set the extract stream value as a label - stage.labels { - values = { - stream = "", - } - } - } - - // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, - // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary - // container runtime label as it is no longer needed. - stage.label_drop { - values = [ - "filename", - "tmp_container_runtime", - ] - } - forward_to = argument.logs_destinations.value - } - } - pod_logs "feature" { - logs_destinations = [ - loki.write.loki.receiver, - ] - } ---- -# Source: k8s-monitoring/templates/alloy-metrics-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -338,6 +200,7 @@ data: } } } + // Feature: Cluster Metrics declare "cluster_metrics" { argument "metrics_destinations" { @@ -538,11 +401,486 @@ data: } cluster_metrics "feature" { metrics_destinations = [ - prometheus.remote_write.prometheus.receiver, + prometheus.remote_write.prometheus.receiver, + ] + } +--- +# Source: k8s-monitoring/templates/alloy-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: ko-alloy-singleton + namespace: default +data: + config.alloy: |- + // Destination: prometheus (prometheus) + otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] + } + + prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + } + proxy_url = "https://myproxy.default.svc:8080" + tls_config { + insecure_skip_verify = false + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "proxies-example-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "proxies-example-cluster" + target_label = "cluster" + } + } + } + // Destination: loki (loki) + otelcol.exporter.loki "loki" { + forward_to = [loki.write.loki.receiver] + } + + loki.write "loki" { + endpoint { + url = "http://loki.loki.svc:3100/loki/api/v1/push" + } + external_labels = { + cluster = "proxies-example-cluster", + "k8s_cluster_name" = "proxies-example-cluster", + } + } + + // Feature: Cluster Events + declare "cluster_events" { + argument "logs_destinations" { + comment = "Must be a list of log destinations where collected logs should be forwarded to" + } + + loki.source.kubernetes_events "cluster_events" { + job_name = "integrations/kubernetes/eventhandler" + log_format = "logfmt" + forward_to = argument.logs_destinations.value + } + } + cluster_events "feature" { + logs_destinations = [ + loki.write.loki.receiver, + ] + } + + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "ko" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "1h" + clustering { + enabled = true + } + forward_to = [ + prometheus.remote_write.prometheus.receiver, + ] + } + + + self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.0-alpha.1", namespace="default", platform=""} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{feature="applicationObservability", protocols="otlpgrpc", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{deployments="kube-state-metrics,node-exporter,windows-exporter", feature="clusterMetrics", sources="kubelet,kubeletResource,cadvisor,kube-state-metrics,node-exporter,windows-exporter", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="clusterEvents", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="podLogs", method="volumes", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="profiling", method="", version="1.0.0"} 1 +--- +# Source: k8s-monitoring/templates/alloy-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: ko-alloy-logs + namespace: default +data: + config.alloy: |- + // Destination: loki (loki) + otelcol.exporter.loki "loki" { + forward_to = [loki.write.loki.receiver] + } + + loki.write "loki" { + endpoint { + url = "http://loki.loki.svc:3100/loki/api/v1/push" + } + external_labels = { + cluster = "proxies-example-cluster", + "k8s_cluster_name" = "proxies-example-cluster", + } + } + + // Feature: Pod Logs + declare "pod_logs" { + argument "logs_destinations" { + comment = "Must be a list of log destinations where collected logs should be forwarded to" + } + + discovery.relabel "filtered_pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "$1" + target_label = "job" + } + + // set the container runtime as a label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_id"] + regex = "^(\\S+):\\/\\/.+$" + replacement = "$1" + target_label = "tmp_container_runtime" + } + } + + discovery.kubernetes "pods" { + role = "pod" + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } + } + + discovery.relabel "filtered_pods_with_paths" { + targets = discovery.relabel.filtered_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "/var/log/pods/*$1/*.log" + target_label = "__path__" + } + } + + local.file_match "pod_logs" { + path_targets = discovery.relabel.filtered_pods_with_paths.output + } + + loki.source.file "pod_logs" { + targets = local.file_match.pod_logs.targets + forward_to = [loki.process.pod_logs.receiver] + } + + loki.process "pod_logs" { + stage.match { + selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" + // the cri processing stage extracts the following k/v pairs: log, stream, time, flags + stage.cri {} + + // Set the extract flags and stream values as labels + stage.labels { + values = { + flags = "", + stream = "", + } + } + } + + stage.match { + selector = "{tmp_container_runtime=\"docker\"}" + // the docker processing stage extracts the following k/v pairs: log, stream, time + stage.docker {} + + // Set the extract stream value as a label + stage.labels { + values = { + stream = "", + } + } + } + + // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, + // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary + // container runtime label as it is no longer needed. + stage.label_drop { + values = [ + "filename", + "tmp_container_runtime", + ] + } + forward_to = argument.logs_destinations.value + } + } + pod_logs "feature" { + logs_destinations = [ + loki.write.loki.receiver, + ] + } +--- +# Source: k8s-monitoring/templates/alloy-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: ko-alloy-receiver + namespace: default +data: + config.alloy: |- + // Destination: prometheus (prometheus) + otelcol.exporter.prometheus "prometheus" { + forward_to = [prometheus.remote_write.prometheus.receiver] + } + + prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.prometheus.svc:9090/api/v1/write" + headers = { + } + proxy_url = "https://myproxy.default.svc:8080" + tls_config { + insecure_skip_verify = false + } + send_native_histograms = false + queue_config { + capacity = 10000 + min_shards = 1 + max_shards = 50 + max_samples_per_send = 2000 + batch_send_deadline = "5s" + min_backoff = "30ms" + max_backoff = "5s" + retry_on_http_429 = true + sample_age_limit = "0s" + } + write_relabel_config { + source_labels = ["cluster"] + regex = "" + replacement = "proxies-example-cluster" + target_label = "cluster" + } + write_relabel_config { + source_labels = ["k8s.cluster.name"] + regex = "" + replacement = "proxies-example-cluster" + target_label = "cluster" + } + } + } + // Destination: loki (loki) + otelcol.exporter.loki "loki" { + forward_to = [loki.write.loki.receiver] + } + + loki.write "loki" { + endpoint { + url = "http://loki.loki.svc:3100/loki/api/v1/push" + } + external_labels = { + cluster = "proxies-example-cluster", + "k8s_cluster_name" = "proxies-example-cluster", + } + } + // Destination: tempo (otlp) + + otelcol.processor.transform "tempo" { + error_mode = "ignore" + metric_statements { + context = "resource" + statements = ["set(attributes[\"k8s.cluster.name\"], \"proxies-example-cluster\") where attributes[\"k8s.cluster.name\"] == nil"] + } + log_statements { + context = "resource" + statements = ["set(attributes[\"k8s.cluster.name\"], \"proxies-example-cluster\") where attributes[\"k8s.cluster.name\"] == nil"] + } + trace_statements { + context = "resource" + statements = ["set(attributes[\"k8s.cluster.name\"], \"proxies-example-cluster\") where attributes[\"k8s.cluster.name\"] == nil"] + } + + output { + metrics = [otelcol.exporter.otlp.tempo.input] + logs = [otelcol.exporter.otlp.tempo.input] + traces = [otelcol.exporter.otlp.tempo.input] + } + } + otelcol.exporter.otlp "tempo" { + client { + endpoint = "http://tempo.tempo.svc:4317" + headers = { + } + tls { + insecure = false + insecure_skip_verify = false + } + } + } + + // Feature: Application Observability + declare "application_observability" { + argument "metrics_destinations" { + comment = "Must be a list of metrics destinations where collected metrics should be forwarded to" + } + + argument "logs_destinations" { + comment = "Must be a list of log destinations where collected logs should be forwarded to" + } + + argument "traces_destinations" { + comment = "Must be a list of trace destinations where collected trace should be forwarded to" + } + + // Receivers --> Resource Detection Processor + otelcol.receiver.otlp "receiver" { + grpc { + endpoint = "0.0.0.0:4317" + } + debug_metrics { + disable_high_cardinality_metrics = true + } + output { + metrics = [otelcol.processor.resourcedetection.default.input] + logs = [otelcol.processor.resourcedetection.default.input] + traces = [otelcol.processor.resourcedetection.default.input] + } + } + + // Resource Detection Processor --> K8s Attribute Processor + otelcol.processor.resourcedetection "default" { + detectors = ["env", "system"] + system { + hostname_sources = ["os"] + } + + output { + metrics = [otelcol.processor.k8sattributes.default.input] + logs = [otelcol.processor.k8sattributes.default.input] + traces = [otelcol.processor.k8sattributes.default.input] + } + } + + // K8s Attribute Processor --> Transform Processor + // Resource Detection Processor Traces --> Host Info Connector + otelcol.processor.k8sattributes "default" { + extract { + metadata = ["k8s.namespace.name","k8s.pod.name","k8s.deployment.name","k8s.statefulset.name","k8s.daemonset.name","k8s.cronjob.name","k8s.job.name","k8s.node.name","k8s.pod.uid","k8s.pod.start_time"] + } + pod_association { + source { + from = "connection" + } + } + + output { + metrics = [otelcol.processor.transform.default.input] + logs = [otelcol.processor.transform.default.input] + traces = [otelcol.processor.transform.default.input, otelcol.connector.host_info.default.input] + } + } + // Host Info Connector --> Batch Processor + otelcol.connector.host_info "default" { + host_identifiers = [ "k8s.node.name" ] + + output { + metrics = [otelcol.processor.batch.default.input] + } + } + + + // Transform Processor --> Batch Processor + otelcol.processor.transform "default" { + error_mode = "ignore" + log_statements { + context = "resource" + statements = [ + "set(attributes[\"pod\"], attributes[\"k8s.pod.name\"])", + "set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])", + "set(attributes[\"loki.resource.labels\"], \"cluster, namespace, job, pod\")", + ] + } + + output { + metrics = [otelcol.processor.batch.default.input] + logs = [otelcol.processor.batch.default.input] + traces = [otelcol.processor.batch.default.input] + } + } + + // Batch Processor --> Destinations + otelcol.processor.batch "default" { + output { + metrics = argument.metrics_destinations.value + logs = argument.logs_destinations.value + traces = argument.traces_destinations.value + } + } + } + application_observability "feature" { + metrics_destinations = [ + otelcol.exporter.prometheus.prometheus.input, + ] + logs_destinations = [ + otelcol.exporter.loki.loki.input, + ] + traces_destinations = [ + otelcol.processor.transform.tempo.input, ] } --- -# Source: k8s-monitoring/templates/alloy-profiles-config.yaml +# Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -1397,256 +1735,6 @@ data: ] } --- -# Source: k8s-monitoring/templates/alloy-receiver-config.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: ko-alloy-receiver - namespace: default -data: - config.alloy: |- - // Destination: prometheus (prometheus) - otelcol.exporter.prometheus "prometheus" { - forward_to = [prometheus.remote_write.prometheus.receiver] - } - - prometheus.remote_write "prometheus" { - endpoint { - url = "http://prometheus.prometheus.svc:9090/api/v1/write" - headers = { - } - proxy_url = "https://myproxy.default.svc:8080" - tls_config { - insecure_skip_verify = false - } - send_native_histograms = false - queue_config { - capacity = 10000 - min_shards = 1 - max_shards = 50 - max_samples_per_send = 2000 - batch_send_deadline = "5s" - min_backoff = "30ms" - max_backoff = "5s" - retry_on_http_429 = true - sample_age_limit = "0s" - } - write_relabel_config { - source_labels = ["cluster"] - regex = "" - replacement = "proxies-example-cluster" - target_label = "cluster" - } - write_relabel_config { - source_labels = ["k8s.cluster.name"] - regex = "" - replacement = "proxies-example-cluster" - target_label = "cluster" - } - } - } - // Destination: loki (loki) - otelcol.exporter.loki "loki" { - forward_to = [loki.write.loki.receiver] - } - - loki.write "loki" { - endpoint { - url = "http://loki.loki.svc:3100/loki/api/v1/push" - } - external_labels = { - cluster = "proxies-example-cluster", - "k8s_cluster_name" = "proxies-example-cluster", - } - } - // Destination: tempo (otlp) - - otelcol.processor.transform "tempo" { - error_mode = "ignore" - metric_statements { - context = "resource" - statements = ["set(attributes[\"k8s.cluster.name\"], \"proxies-example-cluster\") where attributes[\"k8s.cluster.name\"] == nil"] - } - log_statements { - context = "resource" - statements = ["set(attributes[\"k8s.cluster.name\"], \"proxies-example-cluster\") where attributes[\"k8s.cluster.name\"] == nil"] - } - trace_statements { - context = "resource" - statements = ["set(attributes[\"k8s.cluster.name\"], \"proxies-example-cluster\") where attributes[\"k8s.cluster.name\"] == nil"] - } - - output { - metrics = [otelcol.exporter.otlp.tempo.input] - logs = [otelcol.exporter.otlp.tempo.input] - traces = [otelcol.exporter.otlp.tempo.input] - } - } - otelcol.exporter.otlp "tempo" { - client { - endpoint = "http://tempo.tempo.svc:4317" - headers = { - } - tls { - insecure = false - insecure_skip_verify = false - } - } - } - - - // Feature: Application Observability - declare "application_observability" { - argument "metrics_destinations" { - comment = "Must be a list of metrics destinations where collected metrics should be forwarded to" - } - - argument "logs_destinations" { - comment = "Must be a list of log destinations where collected logs should be forwarded to" - } - - argument "traces_destinations" { - comment = "Must be a list of trace destinations where collected trace should be forwarded to" - } - - // Receivers --> Resource Detection Processor - otelcol.receiver.otlp "receiver" { - grpc { - endpoint = "0.0.0.0:4317" - } - debug_metrics { - disable_high_cardinality_metrics = true - } - output { - metrics = [otelcol.processor.resourcedetection.default.input] - logs = [otelcol.processor.resourcedetection.default.input] - traces = [otelcol.processor.resourcedetection.default.input] - } - } - - // Resource Detection Processor --> K8s Attribute Processor - otelcol.processor.resourcedetection "default" { - detectors = ["env", "system"] - system { - hostname_sources = ["os"] - } - - output { - metrics = [otelcol.processor.k8sattributes.default.input] - logs = [otelcol.processor.k8sattributes.default.input] - traces = [otelcol.processor.k8sattributes.default.input] - } - } - - // K8s Attribute Processor --> Transform Processor - // Resource Detection Processor Traces --> Host Info Connector - otelcol.processor.k8sattributes "default" { - extract { - metadata = ["k8s.namespace.name","k8s.pod.name","k8s.deployment.name","k8s.statefulset.name","k8s.daemonset.name","k8s.cronjob.name","k8s.job.name","k8s.node.name","k8s.pod.uid","k8s.pod.start_time"] - } - pod_association { - source { - from = "connection" - } - } - - output { - metrics = [otelcol.processor.transform.default.input] - logs = [otelcol.processor.transform.default.input] - traces = [otelcol.processor.transform.default.input, otelcol.connector.host_info.default.input] - } - } - // Host Info Connector --> Batch Processor - otelcol.connector.host_info "default" { - host_identifiers = [ "k8s.node.name" ] - - output { - metrics = [otelcol.processor.batch.default.input] - } - } - - - // Transform Processor --> Batch Processor - otelcol.processor.transform "default" { - error_mode = "ignore" - log_statements { - context = "resource" - statements = [ - "set(attributes[\"pod\"], attributes[\"k8s.pod.name\"])", - "set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])", - "set(attributes[\"loki.resource.labels\"], \"cluster, namespace, job, pod\")", - ] - } - - output { - metrics = [otelcol.processor.batch.default.input] - logs = [otelcol.processor.batch.default.input] - traces = [otelcol.processor.batch.default.input] - } - } - - // Batch Processor --> Destinations - otelcol.processor.batch "default" { - output { - metrics = argument.metrics_destinations.value - logs = argument.logs_destinations.value - traces = argument.traces_destinations.value - } - } - } - application_observability "feature" { - metrics_destinations = [ - otelcol.exporter.prometheus.prometheus.input, - ] - logs_destinations = [ - otelcol.exporter.loki.loki.input, - ] - traces_destinations = [ - otelcol.processor.transform.tempo.input, - ] - } ---- -# Source: k8s-monitoring/templates/alloy-singleton-config.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: ko-alloy-singleton - namespace: default -data: - config.alloy: |- - // Destination: loki (loki) - otelcol.exporter.loki "loki" { - forward_to = [loki.write.loki.receiver] - } - - loki.write "loki" { - endpoint { - url = "http://loki.loki.svc:3100/loki/api/v1/push" - } - external_labels = { - cluster = "proxies-example-cluster", - "k8s_cluster_name" = "proxies-example-cluster", - } - } - - // Feature: Cluster Events - declare "cluster_events" { - argument "logs_destinations" { - comment = "Must be a list of log destinations where collected logs should be forwarded to" - } - - loki.source.kubernetes_events "cluster_events" { - job_name = "integrations/kubernetes/eventhandler" - log_format = "logfmt" - forward_to = argument.logs_destinations.value - } - } - cluster_events "feature" { - logs_destinations = [ - loki.write.loki.receiver, - ] - } ---- # Source: k8s-monitoring/charts/alloy-logs/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/charts/k8s-monitoring/templates/NOTES.txt b/charts/k8s-monitoring/templates/NOTES.txt index bb62e0a03b..7f56f4a443 100644 --- a/charts/k8s-monitoring/templates/NOTES.txt +++ b/charts/k8s-monitoring/templates/NOTES.txt @@ -1,29 +1,25 @@ Grafana {{.Chart.Name}} Helm chart deployed! This chart will install the following components: -{{- range $feature := (include "features.list" .) | fromYamlArray }} - {{- if eq (include (printf "features.%s.enabled" $feature) (dict "Values" $.Values)) "true" }} - {{- include (printf "feature.%s.notes.deployments" $feature) (dict "Values" (index $.Values $feature)) | indent 0 }} - {{- end }} +{{- range $feature := (include "features.list.enabled" .) | fromYamlArray }} + {{- include (printf "feature.%s.notes.deployments" $feature) (dict "Values" (index $.Values $feature)) | indent 0 }} {{- end }} {{- include "collectors.notes.deployments" . }} It will: -{{- range $feature := (include "features.list" .) | fromYamlArray }} - {{- if eq (include (printf "features.%s.enabled" $feature) (dict "Values" $.Values)) "true" }} +{{- range $feature := (include "features.list.enabled" .) | fromYamlArray }} + {{- if ne $feature "selfReporting" }} {{- $task := include (printf "feature.%s.notes.task" $feature) (dict "Values" (index $.Values $feature)) | trim }} {{- $destinations := include (printf "features.%s.destinations" $feature) (dict "Values" $.Values) | fromYamlArray }} * {{ $task }} and send data to {{ include "english_list" $destinations }}. {{- end }} {{- end }} -{{- range $feature := (include "features.list" .) | fromYamlArray }} - {{- if eq (include (printf "features.%s.enabled" $feature) (dict "Values" $.Values)) "true" }} - {{- $collectorName := (index $.Values $feature).collector }} - {{- $Collector := dict "Namespace" $.Release.Namespace }} - {{- if $collectorName }} - {{- $Collector = merge $Collector (dict "ServiceName" (include "alloy.fullname" (index $.Subcharts $collectorName))) }} - {{- end }} - {{ include (printf "feature.%s.notes.actions" $feature) (dict "Values" (index $.Values $feature) "Collector" $Collector) | indent 0 }} +{{- range $feature := (include "features.list.enabled" .) | fromYamlArray }} + {{- $collectorName := (index $.Values $feature).collector }} + {{- $Collector := dict "Namespace" $.Release.Namespace }} + {{- if $collectorName }} + {{- $Collector = merge $Collector (dict "ServiceName" (include "alloy.fullname" (index $.Subcharts $collectorName))) }} {{- end }} + {{ include (printf "feature.%s.notes.actions" $feature) (dict "Values" (index $.Values $feature) "Collector" $Collector) | indent 0 }} {{- end }} diff --git a/charts/k8s-monitoring/templates/_helpers.tpl b/charts/k8s-monitoring/templates/_helpers.tpl index 3100ac41dc..adf1dd9c33 100644 --- a/charts/k8s-monitoring/templates/_helpers.tpl +++ b/charts/k8s-monitoring/templates/_helpers.tpl @@ -31,3 +31,11 @@ {{- join ", " $rest }}, and {{ $last }} {{- end }} {{- end }} + +{{- define "label_list" }} +{{- $labels := list }} +{{- range $key, $value := . }} + {{- $labels = append $labels (printf "%s=\"%s\"" $key $value) }} +{{- end }} +{{- printf "{%s}" ($labels | join ", ") }} +{{- end }} diff --git a/charts/k8s-monitoring/templates/alloy-config.yaml b/charts/k8s-monitoring/templates/alloy-config.yaml new file mode 100644 index 0000000000..e9cfdaba05 --- /dev/null +++ b/charts/k8s-monitoring/templates/alloy-config.yaml @@ -0,0 +1,37 @@ +{{- range $collector := include "collectors.list.enabled" . | fromYamlArray }} +{{- $selfReportingEnabled := false }} +{{- $destinations := list }} +{{- range $feature := include "features.list" $ | fromYamlArray }} + {{- if has $collector (include (printf "features.%s.collectors" $feature) $ | fromYamlArray ) }} + {{- if eq $feature "selfReporting" }} + {{- $selfReportingEnabled = true }} + {{- end }} + {{- $destinations = concat $destinations ((include (printf "features.%s.destinations" $feature) $) | fromYamlArray) }} + {{- end }} +{{- end }} + +{{/* Save the top level object and add the collector name */}} +{{- $values := deepCopy $ | merge (dict "collectorName" $collector) }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "alloy.fullname" (index $.Subcharts $collector) }} + namespace: {{ $.Release.Namespace }} +data: + config.alloy: |- + {{- include "destinations.alloy.config" (deepCopy $ | merge (dict "names" $destinations)) | trim | nindent 4 }} +{{- range $feature := include "features.list" . | fromYamlArray }} + {{- if has $collector (include (printf "features.%s.collectors" $feature) $ | fromYamlArray ) }} + {{ include (printf "features.%s.include" $feature) $values | trim | nindent 4 }} + {{- end }} +{{- end }} + {{- include "collectors.logging.alloy" $values | trim | nindent 4 }} + {{- include "collectors.liveDebugging.alloy" $values | trim | nindent 4 }} + {{- if (index $.Values $collector).extraConfig }} + {{ (index $.Values $collector).extraConfig | trim | nindent 4 }} + {{- end }} +{{- if $selfReportingEnabled }} + {{- include "features.selfReporting.file" $values | trim | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/k8s-monitoring/templates/alloy-logs-config.yaml b/charts/k8s-monitoring/templates/alloy-logs-config.yaml deleted file mode 100644 index d7db26f66c..0000000000 --- a/charts/k8s-monitoring/templates/alloy-logs-config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- $collectorName := "alloy-logs" }} -{{- if (index .Values $collectorName).enabled }} -{{- $destinations := include "features.podLogs.destinations" . | fromYamlArray }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "alloy.fullname" (index .Subcharts $collectorName) }} - namespace: {{ $.Release.Namespace }} -data: - config.alloy: |- - {{- include "destinations.alloy.config" (dict "destinations" $.Values.destinations "names" $destinations "clusterName" $.Values.cluster.name "Release" $.Release "Chart" $.Chart "Files" $.Files) | trim | nindent 4 }} - - {{- include "features.podLogs.include" . | trim | nindent 4 }} - - {{- include "collectors.logging.alloy" (dict "Values" .Values "collectorName" $collectorName) | trim | nindent 4 }} - {{- include "collectors.liveDebugging.alloy" (dict "Values" .Values "collectorName" $collectorName) | trim | nindent 4 }} - {{- if (index .Values $collectorName).extraConfig }} - {{ (index .Values $collectorName).extraConfig | trim | nindent 4 }} - {{- end }} -{{- end }} diff --git a/charts/k8s-monitoring/templates/alloy-metrics-config.yaml b/charts/k8s-monitoring/templates/alloy-metrics-config.yaml deleted file mode 100644 index e31ae7748b..0000000000 --- a/charts/k8s-monitoring/templates/alloy-metrics-config.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- $collectorName := "alloy-metrics" }} -{{- if (index .Values $collectorName).enabled }} -{{- $destinations := list }} -{{- $destinations = concat $destinations ((include "features.clusterMetrics.destinations" .) | fromYamlArray) }} -{{- $destinations = concat $destinations ((include "features.annotationAutodiscovery.destinations" .) | fromYamlArray) }} -{{- $destinations = concat $destinations ((include "features.prometheusOperatorObjects.destinations" .) | fromYamlArray) }} -{{- $destinations = concat $destinations ((include "features.integrations.destinations.metrics" .) | fromYamlArray) }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "alloy.fullname" (index .Subcharts $collectorName) }} - namespace: {{ $.Release.Namespace }} -data: - config.alloy: |- - {{- include "destinations.alloy.config" (dict "destinations" $.Values.destinations "names" $destinations "clusterName" $.Values.cluster.name "Release" $.Release "Chart" $.Chart "Files" $.Files) | trim | nindent 4 }} - - {{- include "features.clusterMetrics.include" . | trim | nindent 4 }} - {{- include "features.annotationAutodiscovery.include" . | trim | nindent 4 }} - {{- include "features.prometheusOperatorObjects.include" . | trim | nindent 4 }} - {{- include "features.integrations.metrics.include" . | trim | nindent 4 }} - - {{- include "collectors.logging.alloy" (dict "Values" .Values "collectorName" $collectorName) | trim | nindent 4 }} - {{- include "collectors.liveDebugging.alloy" (dict "Values" .Values "collectorName" $collectorName) | trim | nindent 4 }} - {{- if (index .Values $collectorName).extraConfig }} - {{ (index .Values $collectorName).extraConfig | trim | nindent 4 }} - {{- end }} -{{- end }} diff --git a/charts/k8s-monitoring/templates/alloy-profiles-config.yaml b/charts/k8s-monitoring/templates/alloy-profiles-config.yaml deleted file mode 100644 index 35dcc9ff51..0000000000 --- a/charts/k8s-monitoring/templates/alloy-profiles-config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- $collectorName := "alloy-profiles" }} -{{- if (index .Values $collectorName).enabled }} -{{- $destinations := include "features.profiling.destinations" . | fromYamlArray }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "alloy.fullname" (index .Subcharts $collectorName) }} - namespace: {{ $.Release.Namespace }} -data: - config.alloy: |- - {{- include "destinations.alloy.config" (dict "destinations" $.Values.destinations "names" $destinations "clusterName" $.Values.cluster.name "Release" $.Release "Chart" $.Chart "Files" $.Files) | trim | nindent 4 }} - - {{- include "features.profiling.include" . | nindent 4 }} - - {{- include "collectors.logging.alloy" (dict "Values" .Values "collectorName" $collectorName) | trim | nindent 4 }} - {{- include "collectors.liveDebugging.alloy" (dict "Values" .Values "collectorName" $collectorName) | trim | nindent 4 }} - {{- if (index .Values $collectorName).extraConfig }} - {{ (index .Values $collectorName).extraConfig | trim | nindent 4 }} - {{- end }} -{{- end }} diff --git a/charts/k8s-monitoring/templates/alloy-receiver-config.yaml b/charts/k8s-monitoring/templates/alloy-receiver-config.yaml deleted file mode 100644 index f7678f4a14..0000000000 --- a/charts/k8s-monitoring/templates/alloy-receiver-config.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- $collectorName := "alloy-receiver" }} -{{- if (index .Values $collectorName).enabled }} -{{- $destinations := include "features.applicationObservability.destinations" . | fromYamlArray }} -{{- $destinations = concat $destinations ((include "features.frontendObservability.destinations" .) | fromYamlArray) }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "alloy.fullname" (index .Subcharts $collectorName) }} - namespace: {{ $.Release.Namespace }} -data: - config.alloy: |- - {{- include "destinations.alloy.config" (dict "destinations" $.Values.destinations "names" $destinations "clusterName" $.Values.cluster.name "Release" $.Release "Chart" $.Chart "Files" $.Files) | trim | nindent 4 }} - - {{- include "features.applicationObservability.include" . | nindent 4 }} - {{- include "features.frontendObservability.include" . | nindent 4 }} - - {{- include "collectors.logging.alloy" (dict "Values" .Values "collectorName" $collectorName) | trim | nindent 4 }} - {{- include "collectors.liveDebugging.alloy" (dict "Values" .Values "collectorName" $collectorName) | trim | nindent 4 }} - {{- if (index .Values $collectorName).extraConfig }} - {{ (index .Values $collectorName).extraConfig | trim | nindent 4 }} - {{- end }} -{{- end }} diff --git a/charts/k8s-monitoring/templates/alloy-singleton-config.yaml b/charts/k8s-monitoring/templates/alloy-singleton-config.yaml deleted file mode 100644 index bc6eea1e12..0000000000 --- a/charts/k8s-monitoring/templates/alloy-singleton-config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- $collectorName := "alloy-singleton" }} -{{- if (index .Values $collectorName).enabled }} -{{- $destinations := include "features.clusterEvents.destinations" . | fromYamlArray }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "alloy.fullname" (index .Subcharts $collectorName) }} - namespace: {{ $.Release.Namespace }} -data: - config.alloy: |- - {{- include "destinations.alloy.config" (dict "destinations" $.Values.destinations "names" $destinations "clusterName" $.Values.cluster.name "Release" $.Release "Chart" $.Chart "Files" $.Files) | trim | nindent 4 }} - - {{- include "features.clusterEvents.include" . | nindent 4 }} - - {{- include "collectors.logging.alloy" (dict "Values" .Values "collectorName" $collectorName) | trim | nindent 4 }} - {{- include "collectors.liveDebugging.alloy" (dict "Values" .Values "collectorName" $collectorName) | trim | nindent 4 }} - {{- if (index .Values $collectorName).extraConfig }} - {{ (index .Values $collectorName).extraConfig | trim | nindent 4 }} - {{- end }} -{{- end }} diff --git a/charts/k8s-monitoring/templates/collectors/_collector_helpers.tpl b/charts/k8s-monitoring/templates/collectors/_collector_helpers.tpl index f762b8ef69..97e197b472 100644 --- a/charts/k8s-monitoring/templates/collectors/_collector_helpers.tpl +++ b/charts/k8s-monitoring/templates/collectors/_collector_helpers.tpl @@ -6,6 +6,14 @@ - alloy-profiles {{- end }} +{{- define "collectors.list.enabled" -}} +{{- range $collector := ((include "collectors.list" .) | fromYamlArray ) }} + {{- if (index $.Values $collector).enabled }} +- {{ $collector }} + {{- end }} +{{- end }} +{{- end }} + {{/* Inputs: Values (all values), name (collector name), feature (feature name) */}} {{- define "collectors.require_collector" -}} {{- if not (index .Values .name).enabled }} diff --git a/charts/k8s-monitoring/templates/collectors/_collector_notes.tpl b/charts/k8s-monitoring/templates/collectors/_collector_notes.tpl index 074f9f5c1c..e73cea15a1 100644 --- a/charts/k8s-monitoring/templates/collectors/_collector_notes.tpl +++ b/charts/k8s-monitoring/templates/collectors/_collector_notes.tpl @@ -1,13 +1,11 @@ {{- define "collectors.notes.deployments" }} -{{- range $collector := include "collectors.list" . | fromYamlArray }} - {{- if (index $.Values $collector).enabled }} - {{- $type := (index $.Values $collector).controller.type }} - {{- $replicas := (index $.Values $collector).controller.replicas | default 1 | int }} - {{- if ne $type "daemonset" }} +{{- range $collector := include "collectors.list.enabled" . | fromYamlArray }} + {{- $type := (index $.Values $collector).controller.type }} + {{- $replicas := (index $.Values $collector).controller.replicas | default 1 | int }} + {{- if ne $type "daemonset" }} {{- $type = printf "%s, %d replica" $type $replicas }} {{- if gt $replicas 1 }}{{- $type = printf "%ss" $type }}{{- end }} - {{- end }} -* Grafana Alloy "{{ $collector }}" ({{ $type }}) {{- end }} +* Grafana Alloy "{{ $collector }}" ({{ $type }}) {{- end }} {{- end }} diff --git a/charts/k8s-monitoring/templates/destinations/_config.alloy.txt b/charts/k8s-monitoring/templates/destinations/_config.alloy.tpl similarity index 76% rename from charts/k8s-monitoring/templates/destinations/_config.alloy.txt rename to charts/k8s-monitoring/templates/destinations/_config.alloy.tpl index e31c8e23e3..66a49c38aa 100644 --- a/charts/k8s-monitoring/templates/destinations/_config.alloy.txt +++ b/charts/k8s-monitoring/templates/destinations/_config.alloy.tpl @@ -1,5 +1,5 @@ - Returns an alloy-formatted array of destination targets given the name -{{/*Inputs: destinations (array of destination definition), names ([]string), type (string) ecosystem (string) */}} +{{/* Returns an alloy-formatted array of destination targets given the name */}} +{{/* Inputs: destinations (array of destination definition), names ([]string), type (string) ecosystem (string) */}} {{- define "destinations.alloy.targets" -}} {{- range $destination := .destinations }} {{- if (has $destination.name $.names ) }} @@ -13,10 +13,10 @@ {{/* Adds the Alloy components for destinations */}} {{/*Inputs: destinations (array of destination definition), names([]string) clusterName (string), Release (Release object) Chart (Chart object) Files (Files object) */}} {{- define "destinations.alloy.config" }} -{{- range $destination := .destinations }} +{{- range $destination := .Values.destinations }} {{- if (has $destination.name $.names ) }} // Destination: {{ $destination.name }} ({{ $destination.type }}) -{{- include (printf "destinations.%s.alloy" $destination.type) (dict "destination" $destination "clusterName" $.clusterName "Files" $.Files) | indent 0 }} +{{- include (printf "destinations.%s.alloy" $destination.type) (dict "destination" $destination "clusterName" $.Values.cluster.name "Files" $.Files) | indent 0 }} {{- if eq (include "destinations.secret.uses_k8s_secret" $destination) "true" }} {{- include "destinations.secret.alloy" (dict "destination" $destination "Release" $.Release "Chart" $.Chart) | nindent 0 }} diff --git a/charts/k8s-monitoring/templates/destinations/_destination_helpers.tpl b/charts/k8s-monitoring/templates/destinations/_destination_helpers.tpl index 32a37ae798..cd5e602c3c 100644 --- a/charts/k8s-monitoring/templates/destinations/_destination_helpers.tpl +++ b/charts/k8s-monitoring/templates/destinations/_destination_helpers.tpl @@ -8,7 +8,7 @@ {{- end }} {{- end }} -{{/*Inputs: destinations (array of destination definition), type (string), ecosystem (string), filter (list of destination names) */}} +{{/* Inputs: destinations (array of destination definition), type (string), ecosystem (string), filter (list of destination names) */}} {{- define "destinations.get" -}} {{- $destinations := list }} {{- $backupDestinations := list }} diff --git a/charts/k8s-monitoring/templates/features/_feature_annotation_autodiscovery.tpl b/charts/k8s-monitoring/templates/features/_feature_annotation_autodiscovery.tpl index bf073f3854..e0d428dbc3 100644 --- a/charts/k8s-monitoring/templates/features/_feature_annotation_autodiscovery.tpl +++ b/charts/k8s-monitoring/templates/features/_feature_annotation_autodiscovery.tpl @@ -1,4 +1,11 @@ {{- define "features.annotationAutodiscovery.enabled" }}{{ .Values.annotationAutodiscovery.enabled }}{{- end }} + +{{- define "features.annotationAutodiscovery.collectors" }} +{{- if .Values.annotationAutodiscovery.enabled -}} +- {{ .Values.annotationAutodiscovery.collector }} +{{- end }} +{{- end }} + {{- define "features.annotationAutodiscovery.include" }} {{- if .Values.annotationAutodiscovery.enabled -}} {{- $destinations := include "features.annotationAutodiscovery.destinations" . | fromYamlArray }} diff --git a/charts/k8s-monitoring/templates/features/_feature_application_observability.tpl b/charts/k8s-monitoring/templates/features/_feature_application_observability.tpl index 55027c8f97..198f47e17c 100644 --- a/charts/k8s-monitoring/templates/features/_feature_application_observability.tpl +++ b/charts/k8s-monitoring/templates/features/_feature_application_observability.tpl @@ -1,4 +1,11 @@ {{- define "features.applicationObservability.enabled" }}{{ .Values.applicationObservability.enabled }}{{- end }} + +{{- define "features.applicationObservability.collectors" }} +{{- if .Values.applicationObservability.enabled -}} +- {{ .Values.applicationObservability.collector }} +{{- end }} +{{- end }} + {{- define "features.applicationObservability.include" }} {{- if .Values.applicationObservability.enabled -}} {{- $destinations := include "features.applicationObservability.destinations" . | fromYamlArray }} diff --git a/charts/k8s-monitoring/templates/features/_feature_cluster_events.tpl b/charts/k8s-monitoring/templates/features/_feature_cluster_events.tpl index 0f48eff22c..0758348ef5 100644 --- a/charts/k8s-monitoring/templates/features/_feature_cluster_events.tpl +++ b/charts/k8s-monitoring/templates/features/_feature_cluster_events.tpl @@ -1,4 +1,11 @@ {{- define "features.clusterEvents.enabled" }}{{ .Values.clusterEvents.enabled }}{{- end }} + +{{- define "features.clusterEvents.collectors" }} +{{- if .Values.clusterEvents.enabled -}} +- {{ .Values.clusterEvents.collector }} +{{- end }} +{{- end }} + {{- define "features.clusterEvents.include" }} {{- if .Values.clusterEvents.enabled -}} {{- $destinations := include "features.clusterEvents.destinations" . | fromYamlArray }} diff --git a/charts/k8s-monitoring/templates/features/_feature_cluster_metrics.tpl b/charts/k8s-monitoring/templates/features/_feature_cluster_metrics.tpl index 8fffc96069..d605f8fdd1 100644 --- a/charts/k8s-monitoring/templates/features/_feature_cluster_metrics.tpl +++ b/charts/k8s-monitoring/templates/features/_feature_cluster_metrics.tpl @@ -1,4 +1,11 @@ {{- define "features.clusterMetrics.enabled" }}{{ .Values.clusterMetrics.enabled }}{{- end }} + +{{- define "features.clusterMetrics.collectors" }} +{{- if .Values.clusterMetrics.enabled -}} +- {{ .Values.clusterMetrics.collector }} +{{- end }} +{{- end }} + {{- define "features.clusterMetrics.include" }} {{- if .Values.clusterMetrics.enabled -}} {{- $destinations := include "features.clusterMetrics.destinations" . | fromYamlArray }} diff --git a/charts/k8s-monitoring/templates/features/_feature_frontend_observability.tpl b/charts/k8s-monitoring/templates/features/_feature_frontend_observability.tpl index 2d701cff5a..f7b29636ba 100644 --- a/charts/k8s-monitoring/templates/features/_feature_frontend_observability.tpl +++ b/charts/k8s-monitoring/templates/features/_feature_frontend_observability.tpl @@ -1,4 +1,11 @@ {{- define "features.frontendObservability.enabled" }}{{ .Values.frontendObservability.enabled }}{{- end }} + +{{- define "features.frontendObservability.collectors" }} +{{- if .Values.frontendObservability.enabled -}} +- {{ .Values.frontendObservability.collector }} +{{- end }} +{{- end }} + {{- define "features.frontendObservability.include" }} {{- if .Values.frontendObservability.enabled -}} {{- $destinations := include "features.frontendObservability.destinations" . | fromYamlArray }} diff --git a/charts/k8s-monitoring/templates/features/_feature_helpers.tpl b/charts/k8s-monitoring/templates/features/_feature_helpers.tpl index e1befefbfd..e15885f8a2 100644 --- a/charts/k8s-monitoring/templates/features/_feature_helpers.tpl +++ b/charts/k8s-monitoring/templates/features/_feature_helpers.tpl @@ -8,4 +8,13 @@ - profiling - prometheusOperatorObjects - integrations +- selfReporting +{{- end }} + +{{- define "features.list.enabled" }} +{{- range $feature := ((include "features.list" .) | fromYamlArray ) }} + {{- if eq (include (printf "features.%s.enabled" $feature) (dict "Values" $.Values)) "true" }} +- {{ $feature }} + {{- end }} +{{- end }} {{- end }} diff --git a/charts/k8s-monitoring/templates/features/_feature_integrations.tpl b/charts/k8s-monitoring/templates/features/_feature_integrations.tpl index 3b6a8db417..eb30d40e1b 100644 --- a/charts/k8s-monitoring/templates/features/_feature_integrations.tpl +++ b/charts/k8s-monitoring/templates/features/_feature_integrations.tpl @@ -4,6 +4,17 @@ {{- if or $metricIntegrations $logIntegrations }}true{{ else }}false{{ end }} {{- end }} +{{- define "features.integrations.collectors" }} +{{- $metricIntegrations := include "feature.integrations.configured.metrics" (dict "Values" .Values.integrations) | fromYamlArray }} +{{- $logIntegrations := include "feature.integrations.configured.logs" (dict "Values" .Values.integrations) | fromYamlArray }} +{{- if $logIntegrations }} +- {{ .Values.integrations.collectors.metrics }} +{{- end }} +{{- if $logIntegrations }} +- {{ .Values.integrations.collectors.logs }} +{{- end }} +{{- end }} + {{- define "features.integrations.metrics.include" }} {{- $destinations := include "features.integrations.destinations.metrics" . | fromYamlArray }} {{- $integrations := include "feature.integrations.configured.metrics" (dict "Values" .Values.integrations) | fromYamlArray }} diff --git a/charts/k8s-monitoring/templates/features/_feature_pod_logs.tpl b/charts/k8s-monitoring/templates/features/_feature_pod_logs.tpl index 4e8ed64d6d..db4499544b 100644 --- a/charts/k8s-monitoring/templates/features/_feature_pod_logs.tpl +++ b/charts/k8s-monitoring/templates/features/_feature_pod_logs.tpl @@ -1,4 +1,11 @@ {{- define "features.podLogs.enabled" }}{{ .Values.podLogs.enabled }}{{- end }} + +{{- define "features.podLogs.collectors" }} +{{- if .Values.podLogs.enabled -}} +- {{ .Values.podLogs.collector }} +{{- end }} +{{- end }} + {{- define "features.podLogs.include" }} {{- if .Values.podLogs.enabled -}} {{- $destinations := include "features.podLogs.destinations" . | fromYamlArray }} diff --git a/charts/k8s-monitoring/templates/features/_feature_profiling.tpl b/charts/k8s-monitoring/templates/features/_feature_profiling.tpl index 2751f3680d..f16917647d 100644 --- a/charts/k8s-monitoring/templates/features/_feature_profiling.tpl +++ b/charts/k8s-monitoring/templates/features/_feature_profiling.tpl @@ -1,4 +1,11 @@ {{- define "features.profiling.enabled" }}{{ .Values.profiling.enabled }}{{- end }} + +{{- define "features.profiling.collectors" }} +{{- if .Values.profiling.enabled -}} +- {{ .Values.profiling.collector }} +{{- end }} +{{- end }} + {{- define "features.profiling.include" }} {{- if .Values.profiling.enabled -}} {{- $destinations := include "features.profiling.destinations" . | fromYamlArray }} diff --git a/charts/k8s-monitoring/templates/features/_feature_prometheus_operator_obejcts.tpl b/charts/k8s-monitoring/templates/features/_feature_prometheus_operator_obejcts.tpl index 349da50ba7..e1ca57e401 100644 --- a/charts/k8s-monitoring/templates/features/_feature_prometheus_operator_obejcts.tpl +++ b/charts/k8s-monitoring/templates/features/_feature_prometheus_operator_obejcts.tpl @@ -1,4 +1,11 @@ {{- define "features.prometheusOperatorObjects.enabled" }}{{ .Values.prometheusOperatorObjects.enabled }}{{- end }} + +{{- define "features.prometheusOperatorObjects.collectors" }} +{{- if .Values.prometheusOperatorObjects.enabled -}} +- {{ .Values.prometheusOperatorObjects.collector }} +{{- end }} +{{- end }} + {{- define "features.prometheusOperatorObjects.include" }} {{- if .Values.prometheusOperatorObjects.enabled -}} {{- $destinations := include "features.prometheusOperatorObjects.destinations" . | fromYamlArray }} diff --git a/charts/k8s-monitoring/templates/features/_feature_self_reporting.tpl b/charts/k8s-monitoring/templates/features/_feature_self_reporting.tpl new file mode 100644 index 0000000000..4642f98604 --- /dev/null +++ b/charts/k8s-monitoring/templates/features/_feature_self_reporting.tpl @@ -0,0 +1,83 @@ +{{- define "features.selfReporting.enabled" -}} +{{- $metricsDestinations := include "destinations.get" (dict "destinations" $.Values.destinations "type" "metrics" "ecosystem" "otlp" "filter" $.Values.applicationObservability.destinations) | fromYamlArray -}} +{{ and .Values.selfReporting.enabled (not (empty $metricsDestinations)) }} +{{- end -}} + +{{- define "features.selfReporting.collectors" -}} +{{- if eq (include "features.selfReporting.enabled" .) "true" }} + {{- $collectorsByIncreasingPreference := list "alloy-receiver" "alloy-metrics" "alloy-singleton" }} + {{- $chosenCollector := "" }} + {{- range $collector := $collectorsByIncreasingPreference }} + {{- if (index $.Values $collector).enabled }}{{- $chosenCollector = $collector }}{{- end -}} + {{- end -}} +- {{ $chosenCollector }} + {{- end -}} +{{- end }} + +{{- define "features.selfReporting.destinations" }} +{{- if eq (include "features.selfReporting.enabled" .) "true" }} +{{- include "destinations.get" (dict "destinations" $.Values.destinations "type" "metrics" "ecosystem" "prometheus" "filter" $.Values.clusterMetrics.destinations) -}} +{{- end }} +{{- end }} + +{{- define "features.selfReporting.validate" }}{{ end }} +{{- define "features.selfReporting.include" }} +{{- if eq (include "features.selfReporting.enabled" .) "true" }} +{{- $destinations := include "destinations.get" (dict "destinations" $.Values.destinations "type" "metrics" "ecosystem" "otlp" "filter" $.Values.applicationObservability.destinations) | fromYamlArray -}} + +// Self Reporting +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } +} + +discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "{{ .Release.Name }}" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = {{ .Values.selfReporting.scrapeInterval | default "1h" | quote}} + clustering { + enabled = true + } + forward_to = [ + {{ include "destinations.alloy.targets" (dict "destinations" $.Values.destinations "names" $destinations "type" "metrics" "ecosystem" "prometheus") | indent 4 | trim }} + ] +} +{{- end }} +{{- end }} + +{{- define "features.selfReporting.file" }} +{{- if eq (include "features.selfReporting.enabled" .) "true" }} +self-reporting-metric.prom: | + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="{{ .Chart.Version }}", namespace="{{ .Release.Namespace }}", platform="{{ .Values.global.platform }}"} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge +{{- range $feature := include "features.list.enabled" . | fromYamlArray }} + {{- if ne $feature "selfReporting" }} + {{- $featureSummary := include (printf "feature.%s.summary" $feature) (dict "Chart" (index $.Subcharts $feature).Chart "Values" (index $.Values $feature)) | fromYaml }} + grafana_kubernetes_monitoring_feature_info{{ include "label_list" (merge $featureSummary (dict "feature" $feature)) }} 1 + {{- end }} + {{- end }} +{{- end }} +{{- end }} + +{{- define "feature.selfReporting.notes.deployments" }}{{ end }} +{{- define "feature.selfReporting.notes.task" }}{{ end }} +{{- define "feature.selfReporting.notes.actions" }}{{ end }} diff --git a/charts/k8s-monitoring/templates/validations.yaml b/charts/k8s-monitoring/templates/validations.yaml index dca468cb7e..e35bccfe2c 100644 --- a/charts/k8s-monitoring/templates/validations.yaml +++ b/charts/k8s-monitoring/templates/validations.yaml @@ -10,7 +10,7 @@ {{- include "destinations.validate" . -}} {{- include "collectors.validate.featuresEnabled" . }} -{{- range $collectorName := ((include "collectors.list" .) | fromYamlArray) }} +{{- range $collectorName := ((include "collectors.list.enabled" .) | fromYamlArray) }} {{- include "collectors.validate.liveDebugging" (dict "collectorName" $collectorName "Values" $.Values) }} {{- end }} --- \ No newline at end of file diff --git a/charts/k8s-monitoring/tests/cluster_events_test.yaml b/charts/k8s-monitoring/tests/cluster_events_test.yaml index 41f0f347eb..2923971f74 100644 --- a/charts/k8s-monitoring/tests/cluster_events_test.yaml +++ b/charts/k8s-monitoring/tests/cluster_events_test.yaml @@ -1,7 +1,7 @@ # yamllint disable rule:document-start rule:line-length rule:trailing-spaces suite: Cluster Events feature tests templates: - - alloy-singleton-config.yaml + - alloy-config.yaml - validations.yaml tests: - it: requires a log destination @@ -39,7 +39,7 @@ tests: enabled: true - it: renders the config to gather Kubernetes Cluster Events - template: alloy-singleton-config.yaml + template: alloy-config.yaml set: cluster: name: ci-test-cluster @@ -91,7 +91,7 @@ tests: } - it: prefers a Loki destination, over an OTLP one - template: alloy-singleton-config.yaml + template: alloy-config.yaml set: cluster: name: ci-test-cluster @@ -118,7 +118,7 @@ tests: pattern: otelcol.receiver.loki.otlp.receiver - it: will use the OTLP destination, if it's the only option - template: alloy-singleton-config.yaml + template: alloy-config.yaml set: cluster: name: ci-test-cluster @@ -139,7 +139,7 @@ tests: pattern: otelcol.receiver.loki.otlp.receiver - it: will use the OTLP destination, if it's told to - template: alloy-singleton-config.yaml + template: alloy-config.yaml set: cluster: name: ci-test-cluster @@ -167,7 +167,7 @@ tests: pattern: loki.write.loki.receiver - it: will use both, if it's told to - template: alloy-singleton-config.yaml + template: alloy-config.yaml set: cluster: name: ci-test-cluster diff --git a/charts/k8s-monitoring/tests/integration/cluster-monitoring/test-values.yaml b/charts/k8s-monitoring/tests/integration/cluster-monitoring/test-values.yaml index a6a1c17ab6..252d1bf25d 100644 --- a/charts/k8s-monitoring/tests/integration/cluster-monitoring/test-values.yaml +++ b/charts/k8s-monitoring/tests/integration/cluster-monitoring/test-values.yaml @@ -9,6 +9,16 @@ tests: LOKI_USER: loki LOKI_PASS: lokipassword queries: + # Self reporting metrics + - query: grafana_kubernetes_monitoring_build_info{cluster="cluster-monitoring-feature-test"} + type: promql + - query: grafana_kubernetes_monitoring_feature_info{cluster="cluster-monitoring-feature-test", feature="clusterMetrics"} + type: promql + - query: grafana_kubernetes_monitoring_feature_info{cluster="cluster-monitoring-feature-test", feature="clusterEvents"} + type: promql + - query: grafana_kubernetes_monitoring_feature_info{cluster="cluster-monitoring-feature-test", feature="podLogs", method="volumes"} + type: promql + # Cluster metrics - query: kubernetes_build_info{cluster="cluster-monitoring-feature-test", job="integrations/kubernetes/kubelet"} type: promql diff --git a/charts/k8s-monitoring/tests/integration/cluster-monitoring/values.yaml b/charts/k8s-monitoring/tests/integration/cluster-monitoring/values.yaml index c8dcd9b0a8..1e4644b6bf 100644 --- a/charts/k8s-monitoring/tests/integration/cluster-monitoring/values.yaml +++ b/charts/k8s-monitoring/tests/integration/cluster-monitoring/values.yaml @@ -31,6 +31,8 @@ clusterEvents: podLogs: enabled: true +selfReporting: {scrapeInterval: 1m} # Force self-report to be generated within test time + alloy-metrics: enabled: true diff --git a/charts/k8s-monitoring/tests/integration/control-plane-monitoring/test-values.yaml b/charts/k8s-monitoring/tests/integration/control-plane-monitoring/test-values.yaml index c7c0a099cb..a012b39dbf 100644 --- a/charts/k8s-monitoring/tests/integration/control-plane-monitoring/test-values.yaml +++ b/charts/k8s-monitoring/tests/integration/control-plane-monitoring/test-values.yaml @@ -7,6 +7,16 @@ tests: LOKI_USER: loki LOKI_PASS: lokipassword queries: + # Self reporting metrics + - query: grafana_kubernetes_monitoring_build_info{cluster="control-plane-monitoring-test"} + type: promql + - query: grafana_kubernetes_monitoring_feature_info{cluster="control-plane-monitoring-test", feature="clusterMetrics"} + type: promql + - query: grafana_kubernetes_monitoring_feature_info{cluster="control-plane-monitoring-test", feature="clusterEvents"} + type: promql + - query: grafana_kubernetes_monitoring_feature_info{cluster="control-plane-monitoring-test", feature="podLogs", method="volumes"} + type: promql + # Cluster metrics - query: kubernetes_build_info{cluster="control-plane-monitoring-test", job="integrations/kubernetes/kubelet"} type: promql diff --git a/charts/k8s-monitoring/tests/integration/control-plane-monitoring/values.yaml b/charts/k8s-monitoring/tests/integration/control-plane-monitoring/values.yaml index 413e2abe9e..9992131ed6 100644 --- a/charts/k8s-monitoring/tests/integration/control-plane-monitoring/values.yaml +++ b/charts/k8s-monitoring/tests/integration/control-plane-monitoring/values.yaml @@ -26,6 +26,8 @@ clusterEvents: podLogs: enabled: true +selfReporting: {scrapeInterval: 1m} # Force self-report to be generated within test time + alloy-metrics: enabled: true diff --git a/charts/k8s-monitoring/values.schema.json b/charts/k8s-monitoring/values.schema.json index 9393af0023..025151c629 100644 --- a/charts/k8s-monitoring/values.schema.json +++ b/charts/k8s-monitoring/values.schema.json @@ -733,6 +733,17 @@ "type": "boolean" } } + }, + "selfReporting": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "scrapeInterval": { + "type": "string" + } + } } }, "definitions": { diff --git a/charts/k8s-monitoring/values.yaml b/charts/k8s-monitoring/values.yaml index 9ac7c1d595..d5de7adb9a 100644 --- a/charts/k8s-monitoring/values.yaml +++ b/charts/k8s-monitoring/values.yaml @@ -121,6 +121,9 @@ annotationAutodiscovery: # @section -- Features - Annotation Autodiscovery destinations: [] + # -- Which collector to assign this feature to. Do not change this unless you are sure of what you are doing. + # @section -- Features - Annotation Autodiscovery + # @ignored collector: alloy-metrics # -- Prometheus Operator Objects enables the gathering of metrics from objects like Probes, PodMonitors, and @@ -203,6 +206,20 @@ integrations: logs: alloy-logs +# -- Self-reporting creates a single metric and log that reports anonymized information about how this Helm chart was +# configured. It reports features enabled, destinations types used, and alloy instances enabled. It does not report any +# actual telemetry data, credentials or configuration, or send any data to any destination other than the ones +# configured above. +# @section -- Features - Self-reporting +selfReporting: + # -- Enable Self-reporting. + # @section -- Features - Self-reporting + enabled: true + + # -- How frequently to generate self-report metrics. This does utilize the global scrapeInterval setting. + # @section -- Features - Self-reporting + scrapeInterval: 1h + # # Collectors (Alloy instances) # diff --git a/scripts/lint-yaml.sh b/scripts/lint-yaml.sh index e2a6ba1b0f..8e8baaa1c1 100755 --- a/scripts/lint-yaml.sh +++ b/scripts/lint-yaml.sh @@ -9,7 +9,6 @@ heading "Kubernetes Monitoring Helm" "Onboarding - Performing YAML Linting using # make sure yamllint exists if [[ "$(command -v yamllint || true)" = "" ]]; then emergency "yamllint is required if running lint locally, see (https://pypi.org/project/yamllint/) or run: brew install yamllint"; - exit 1; fi # determine whether or not the script is called directly or sourced