fix: allow opting-into upstream probes

Allow users to opt-into upstream probe definitions.

Signed-off-by: Pranshu Srivastava <[email protected]>

Author: rexagod

jsonnet/kube-prometheus/components/kube-rbac-proxy.libsonnet

@@ -38,6 +38,13 @@ local defaults = {
     'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305',
     'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305',
   ],
+  // Corresponds to KRP's --ignore-paths flag.
+  // Some components (for e.g., KSM) may utilize the flag to allow for communication with external parties in scenarios
+  // where the originating request(s) cannot be modified to the proxy's expectations, and thus, are passed through, as
+  // is, to certain endpoints that they target, without the proxy's intervention. The kubelet, in KSM's case, can thus
+  // query health probe endpoints without being blocked by KRP, thus allowing for http-based probes over exec-based
+  // ones.
+  ignorePaths:: [],
 };
 
 
@@ -50,10 +57,11 @@ function(params) {
   name: krp._config.name,
   image: krp._config.image,
   args: [
-    '--secure-listen-address=' + krp._config.secureListenAddress,
-    '--tls-cipher-suites=' + std.join(',', krp._config.tlsCipherSuites),
-    '--upstream=' + krp._config.upstream,
-  ],
+          '--secure-listen-address=' + krp._config.secureListenAddress,
+          '--tls-cipher-suites=' + std.join(',', krp._config.tlsCipherSuites),
+          '--upstream=' + krp._config.upstream,
+        ]  // Optionals.
+        + if std.length(krp._config.ignorePaths) > 0 then ['--ignore-paths=' + std.join(',', krp._config.ignorePaths)] else defaults.ignorePaths,
   resources: krp._config.resources,
   ports: krp._config.ports,
   securityContext: {

jsonnet/kube-prometheus/components/kube-state-metrics.libsonnet

@@ -15,12 +15,18 @@ local defaults = {
   },
 
   kubeRbacProxyMain:: {
+    ports: [
+      { name: 'http-metrics', containerPort: 8443 },
+    ],
     resources+: {
       limits+: { cpu: '40m' },
       requests+: { cpu: '20m' },
     },
   },
   kubeRbacProxySelf:: {
+    ports: [
+      { name: 'telemetry', containerPort: 9443 },
+    ],
     resources+: {
       limits+: { cpu: '20m' },
       requests+: { cpu: '10m' },
@@ -46,6 +52,8 @@ local defaults = {
       runbookURLPattern: 'https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/%s',
     },
   },
+  // `enableProbes` allows users to opt-into upstream definitions for health probes.
+  enableProbes:: false,
 };
 
 function(params) (import 'github.com/kubernetes/kube-state-metrics/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet') {
@@ -91,14 +99,14 @@ function(params) (import 'github.com/kubernetes/kube-state-metrics/jsonnet/kube-
     spec+: {
       ports: [
         {
-          name: 'https-main',
-          port: 8443,
-          targetPort: 'https-main',
+          name: defaults.kubeRbacProxyMain.ports[0].name,
+          port: defaults.kubeRbacProxyMain.ports[0].containerPort,
+          targetPort: defaults.kubeRbacProxyMain.ports[0].name,
         },
         {
-          name: 'https-self',
-          port: 9443,
-          targetPort: 'https-self',
+          name: defaults.kubeRbacProxySelf.ports[0].name,
+          port: defaults.kubeRbacProxySelf.ports[0].containerPort,
+          targetPort: defaults.kubeRbacProxySelf.ports[0].name,
         },
       ],
     },
@@ -107,21 +115,19 @@ function(params) (import 'github.com/kubernetes/kube-state-metrics/jsonnet/kube-
   local kubeRbacProxyMain = krp(ksm._config.kubeRbacProxyMain {
     name: 'kube-rbac-proxy-main',
     upstream: 'http://127.0.0.1:8081/',
-    secureListenAddress: ':8443',
-    ports: [
-      { name: 'https-main', containerPort: 8443 },
-    ],
+    secureListenAddress: ':' + std.toString(defaults.kubeRbacProxyMain.ports[0].containerPort),
     image: ksm._config.kubeRbacProxyImage,
+    // When enabling probes, kube-rbac-proxy needs to always allow the /livez endpoint.
+    ignorePaths: if ksm._config.enableProbes then ['/livez'] else super.ignorePaths,
   }),
 
   local kubeRbacProxySelf = krp(ksm._config.kubeRbacProxySelf {
     name: 'kube-rbac-proxy-self',
     upstream: 'http://127.0.0.1:8082/',
-    secureListenAddress: ':9443',
-    ports: [
-      { name: 'https-self', containerPort: 9443 },
-    ],
+    secureListenAddress: ':' + std.toString(defaults.kubeRbacProxySelf.ports[0].containerPort),
     image: ksm._config.kubeRbacProxyImage,
+    // When enabling probes, kube-rbac-proxy needs to always allow the /readyz endpoint.
+    ignorePaths: if ksm._config.enableProbes then ['/readyz'] else super.ignorePaths,
   }),
 
   networkPolicy: {
@@ -161,14 +167,31 @@ function(params) (import 'github.com/kubernetes/kube-state-metrics/jsonnet/kube-
         spec+: {
           automountServiceAccountToken: true,
           containers: std.map(function(c) c {
-            ports:: null,
-            livenessProbe:: null,
-            readinessProbe:: null,
             securityContext+: {
               runAsGroup: 65534,
             },
             args: ['--host=127.0.0.1', '--port=8081', '--telemetry-host=127.0.0.1', '--telemetry-port=8082'],
             resources: ksm._config.resources,
+          } + if !ksm._config.enableProbes then {
+            ports:: null,
+            livenessProbe:: null,
+            readinessProbe:: null,
+          } else {
+            ports: defaults.kubeRbacProxyMain.ports + defaults.kubeRbacProxySelf.ports,
+            livenessProbe: {
+              httpGet: {
+                path: '/livez',
+                port: defaults.kubeRbacProxyMain.ports[0].name,
+                scheme: 'HTTPS',
+              },
+            },
+            readinessProbe: {
+              httpGet: {
+                path: '/readyz',
+                port: defaults.kubeRbacProxySelf.ports[0].name,
+                scheme: 'HTTPS',
+              },
+            },
           }, super.containers) + [kubeRbacProxyMain, kubeRbacProxySelf],
         },
       },

manifests/kubeStateMetrics-deployment.yaml

@@ -60,7 +60,7 @@ spec:
         name: kube-rbac-proxy-main
         ports:
         - containerPort: 8443
-          name: https-main
+          name: http-metrics
         resources:
           limits:
             cpu: 40m
@@ -87,7 +87,7 @@ spec:
         name: kube-rbac-proxy-self
         ports:
         - containerPort: 9443
-          name: https-self
+          name: telemetry
         resources:
           limits:
             cpu: 20m

manifests/kubeStateMetrics-service.yaml

@@ -11,12 +11,12 @@ metadata:
 spec:
   clusterIP: None
   ports:
-  - name: https-main
+  - name: http-metrics
     port: 8443
-    targetPort: https-main
-  - name: https-self
+    targetPort: http-metrics
+  - name: telemetry
     port: 9443
-    targetPort: https-self
+    targetPort: telemetry
   selector:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics