Create the option for loading alloy modules embedded in the chart, rather than from git (#843)

Signed-off-by: Pete Wall <[email protected]>

Author: petewall

CONTRIBUTING.md

@@ -31,6 +31,7 @@ installing them for a better experience:
 -   [helm-docs](https://github.com/norwoodj/helm-docs) - Used for generating Helm chart README.md files.
 -   [helm unittest](https://github.com/helm-unittest/helm-unittest) - Used for executing Helm chart unit tests.
 -   [shellspec](https://github.com/shellspec/shellspec) - Used for executing some unit tests.
+-   [vendir](https://carvel.dev/vendir/) - Used for downloading Alloy Module
 
 Each chart has a Makefile with targets to automate much of the process.
 

charts/feature-annotation-autodiscovery/templates/_module.alloy.tpl

@@ -230,3 +230,5 @@ declare "annotation_autodiscovery" {
   }
 }
 {{- end -}}
+
+{{- define "feature.annotationAutodiscovery.alloyModules" }}{{- end }}

charts/feature-application-observability/templates/_module.alloy.tpl

@@ -82,3 +82,5 @@ declare "application_observability" {
 {{- end }}
 }
 {{- end }}
+
+{{- define "feature.applicationObservability.alloyModules" }}{{- end }}

charts/feature-cluster-events/templates/_module.alloy.tpl

@@ -20,4 +20,6 @@ declare "cluster_events" {
     forward_to = argument.logs_destinations.value
   }
 }
-{{- end -}}
+{{- end -}}
+
+{{- define "feature.clusterEvents.alloyModules" }}{{- end }}

charts/feature-cluster-metrics/README.md

@@ -214,6 +214,8 @@ Actual integration testing in a live environment should be done in the main [k8s
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
+| global.alloyModules.branch | string | `"main"` | If using git, the branch of the git repository to use. |
+| global.alloyModules.source | string | `"git"` | The source of the Alloy modules. The valid options are "configMap" or "git" |
 | global.maxCacheSize | int | `100000` | Sets the max_cache_size for every prometheus.relabel component. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#arguments)) This should be at least 2x-5x your largest scrape target or samples appended rate. |
 | global.platform | string | `""` | The specific platform for this cluster. Will enable compatibility for some platforms. Supported options: (empty) or "openshift". |
 | global.scrapeInterval | string | `"60s"` | How frequently to scrape metrics. |

charts/feature-cluster-metrics/templates/_helpers_modules.tpl

@@ -0,0 +1,24 @@
+{{/* Loads a module from Alloy Modules*/}}
+{{/* Inputs: . ($), name (string), path (string)*/}}
+{{- define "alloyModules.load" }}
+{{- if eq .Values.global.alloyModules.source "configMap" }}
+{{- $pathParts := regexSplit "/" .path -1 }}
+{{- $configMapName := printf "%s-alloy-module-%s" $.Release.Name (index $pathParts 1) }}
+{{- $moduleFile := (slice $pathParts 2) | join "_" }}
+remote.kubernetes.configmap {{ .name | quote }} {
+  name = {{ $configMapName | quote }}
+  namespace = {{ .Release.Namespace | quote }}
+}
+
+import.string {{ .name | quote }} {
+  content = remote.kubernetes.configmap.{{ .name }}.data[{{ $moduleFile | quote }}]
+}
+{{- else if eq .Values.global.alloyModules.source "git" }}
+import.git {{ .name | quote }} {
+  repository = "https://github.com/grafana/alloy-modules.git"
+  revision = "main"
+  path = {{ .path | quote }}
+  pull_frequency = "15m"
+}
+{{- end }}
+{{- end }}

charts/feature-cluster-metrics/templates/_kube_state_metrics.alloy.tpl

@@ -11,13 +11,7 @@
 {{- if (index .Values "kube-state-metrics").enabled }}
 {{- $metricAllowList := include "feature.clusterMetrics.kube_state_metrics.allowList" . }}
 {{- $metricDenyList := (index .Values "kube-state-metrics").metricsTuning.excludeMetrics }}
-
-import.git "kube_state_metrics" {
-  repository = "https://github.com/grafana/alloy-modules.git"
-  revision = "main"
-  path = "modules/kubernetes/kube-state-metrics/metrics.alloy"
-  pull_frequency = "15m"
-}
+{{- include "alloyModules.load" (deepCopy $ | merge (dict "name" "kube_state_metrics" "path" "modules/kubernetes/kube-state-metrics/metrics.alloy")) | nindent 0 }}
 
 kube_state_metrics.kubernetes "targets" {
   label_selectors = [

charts/feature-cluster-metrics/templates/_module.alloy.tpl

@@ -5,12 +5,7 @@ declare "cluster_metrics" {
   }
 
   {{- if or .Values.cadvisor.enabled .Values.kubelet.enabled .Values.kubeletResource.enabled (or .Values.apiServer.enabled (and .Values.controlPlane.enabled (not (eq .Values.apiServer.enabled false)))) }}
-  import.git "kubernetes" {
-    repository = "https://github.com/grafana/alloy-modules.git"
-    revision = "main"
-    path = "modules/kubernetes/core/metrics.alloy"
-    pull_frequency = "15m"
-  }
+  {{- include "alloyModules.load" (deepCopy $ | merge (dict "name" "kubernetes" "path" "modules/kubernetes/core/metrics.alloy")) | nindent 2 }}
   {{- end }}
   {{- include "feature.clusterMetrics.kubelet.alloy" . | indent 2 }}
   {{- include "feature.clusterMetrics.kubeletResource.alloy" . | indent 2 }}
@@ -24,4 +19,16 @@ declare "cluster_metrics" {
   {{- include "feature.clusterMetrics.windows_exporter.alloy" . | indent 2 }}
   {{- include "feature.clusterMetrics.kepler.alloy" . | indent 2 }}
 }
-{{- end -}}
+{{- end -}}
+
+{{- define "feature.clusterMetrics.alloyModules" }}
+{{- if or .Values.cadvisor.enabled .Values.kubelet.enabled .Values.kubeletResource.enabled (or .Values.apiServer.enabled (and .Values.controlPlane.enabled (not (eq .Values.apiServer.enabled false)))) }}
+- modules/kubernetes/core/metrics.alloy
+{{- end }}
+{{- if (index .Values "kube-state-metrics").enabled }}
+- modules/kubernetes/kube-state-metrics/metrics.alloy
+{{- end }}
+{{- if (index .Values "node-exporter").enabled }}
+- modules/system/node-exporter/metrics.alloy
+{{- end }}
+{{- end }}

charts/feature-cluster-metrics/templates/_node_exporter.alloy.tpl

@@ -14,13 +14,7 @@
 {{- if (index .Values "node-exporter").enabled }}
 {{- $metricAllowList := include "feature.clusterMetrics.node_exporter.allowList" . }}
 {{- $metricDenyList := (index .Values "node-exporter").metricsTuning.excludeMetrics }}
-
-import.git "node_exporter" {
-  repository = "https://github.com/grafana/alloy-modules.git"
-  revision = "main"
-  path = "modules/system/node-exporter/metrics.alloy"
-  pull_frequency = "15m"
-}
+{{- include "alloyModules.load" (deepCopy $ | merge (dict "name" "node_exporter" "path" "modules/system/node-exporter/metrics.alloy")) | nindent 0 }}
 
 node_exporter.kubernetes "targets" {
   label_selectors = [

charts/feature-cluster-metrics/templates/_windows_exporter.alloy.tpl

@@ -12,29 +12,31 @@
 
 {{- define "feature.clusterMetrics.windows_exporter.alloy" }}
 {{- if (index .Values "windows-exporter").enabled }}
-{{- $metricAllowList := include "feature.clusterMetrics.windows_exporter.allowList" . }}
+{{- $metricAllowList := include "feature.clusterMetrics.windows_exporter.allowList" . | fromYamlArray }}
 {{- $metricDenyList := (index .Values "windows-exporter").metricsTuning.excludeMetrics }}
-
-import.git "windows_exporter" {
-  repository = "https://github.com/grafana/alloy-modules.git"
-  revision = "main"
-  path = "modules/system/node-exporter/metrics.alloy"
-  pull_frequency = "15m"
-}
-
-windows_exporter.kubernetes "targets" {
-  label_selectors = [
+{{- $labelSelectors := list }}
 {{- range $label, $value := (index .Values "windows-exporter").labelMatchers }}
-    {{ printf "%s=%s" $label $value | quote }},
+  {{- $labelSelectors = append $labelSelectors (printf "%s=%s" $label $value) }}
 {{- end }}
 {{- if (index .Values "windows-exporter").deploy }}
-    {{ printf "release=%s" .Release.Name | quote }},
+  {{- $labelSelectors = append $labelSelectors (printf "release=%s" .Release.Name) }}
+{{- end }}
+
+discovery.kubernetes "windows_exporter_pods" {
+  role = "pod"
+{{- if (index .Values "windows-exporter").deploy }}
+  namespaces {
+    names = [{{ .Release.Namespace | quote }}]
+  }
 {{- end }}
-  ]
+  selectors {
+    role = "pod"
+    label = {{ $labelSelectors | join "," | quote }}
+  }
 }
 
 discovery.relabel "windows_exporter" {
-  targets = windows_exporter.kubernetes.targets.output
+  targets = discovery.kubernetes.windows_exporter_pods.targets
   rule {
     source_labels = ["__meta_kubernetes_pod_node_name"]
     action = "replace"
@@ -45,26 +47,36 @@ discovery.relabel "windows_exporter" {
 {{- end }}
 }
 
-windows_exporter.scrape "metrics" {
-  targets = discovery.relabel.windows_exporter.output
-  clustering = true
-{{- if $metricAllowList }}
-  keep_metrics = "up|{{ $metricAllowList | fromYamlArray | join "|" }}"
-{{- end }}
-{{- if $metricDenyList }}
-  drop_metrics = {{ $metricDenyList | join "|" | quote }}
-{{- end }}
+prometheus.scrape "windows_exporter" {
+  job_name   = "integrations/windows-exporter"
+  targets  = discovery.relabel.windows_exporter.output
   scrape_interval = {{ (index .Values "windows-exporter").scrapeInterval | default .Values.global.scrapeInterval | quote }}
-  max_cache_size = {{ (index .Values "windows-exporter").maxCacheSize | default .Values.global.maxCacheSize | int }}
-{{- if (index .Values "windows-exporter").extraMetricProcessingRules }}
+  clustering {
+    enabled = true
+  }
+{{- if or $metricAllowList $metricDenyList (index .Values "windows-exporter").extraMetricProcessingRules }}
   forward_to = [prometheus.relabel.windows_exporter.receiver]
 }
 
 prometheus.relabel "windows_exporter" {
   max_cache_size = {{ (index .Values "windows-exporter").maxCacheSize | default .Values.global.maxCacheSize | int }}
-
-  {{(index .Values "windows-exporter").extraMetricProcessingRules}}
-
+{{- if $metricAllowList }}
+  rule {
+    source_labels = ["__name__"]
+    regex = "up|{{ $metricAllowList | join "|" }}"
+    action = "keep"
+  }
+{{- end }}
+{{- if $metricDenyList }}
+  rule {
+    source_labels = ["__name__"]
+    regex = {{ $metricDenyList | join "|" | quote }}
+    action = "drop"
+  }
+{{- end }}
+{{- if (index .Values "windows-exporter").extraMetricProcessingRules }}
+{{ (index .Values "windows-exporter").extraMetricProcessingRules | indent 2 }}
+{{- end }}
 {{- end }}
   forward_to = argument.metrics_destinations.value
 }