OCPBUGS-32510: change metrics-server probes for SNO

This change switches the metrics-server's readiness probe to use the
`/livez` endpoint instead of `/readyz` for single-node deployments.

It also adds a startup probe using the same characteristics as the
default readiness probe to ensure that the pod reports ready only when
it has gathered enough samples from kubelet.

By default, the `/readyz` endpoint is used to assert the component
readiness. This endpoint returns success when the metrics-server has
metric samples over 2 intervals (e.g. it has scraped at least one
kubelet twice).

In single-node deployments, it happens sometimes (especially in
end-to-end tests) that the kubelet fails to respond in a timely fashion
due to contention in cAdvisor, leading to a delayed readiness (and test
failures). To workaround the issue, we use the `/livez` endpoint in this
mode.

The long-term plan is to switch resource metrics from cAdvisor to the
CRI stats API (currently an alpha feature). Once it happens, we can
remove this change.

Signed-off-by: Simon Pasquier <[email protected]>

Author: simonpasquier

pkg/manifests/manifests.go

@@ -2053,6 +2053,24 @@ func (f *Factory) MetricsServerDeployment(apiAuthSecretName string, kubeletCABun
 	containers[idx].Args = f.setTLSSecurityConfiguration(podSpec.Containers[0].Args,
 		MetricsServerTLSCipherSuitesFlag, MetricsServerTLSMinTLSVersionFlag)
 
+	// By default, the /readyz endpoint is used to assert the component
+	// readiness. This endpoint returns success when the metrics-server has
+	// metric samples over 2 intervals (e.g. it has scraped at least one
+	// kubelet twice).
+	// In single-node deployments, it happens sometimes (especially in
+	// end-to-end tests) that the kubelet fails to respond in a timely fashion
+	// due to contention in cAdvisor, leading to a delayed readiness (and test
+	// failures). To workaround the issue, we use the /livez endpoint in this
+	// mode.
+	// The long-term plan is to switch resource metrics from cAdvisor to the
+	// CRI stats API (currently an alpha feature). Once it happens, we can
+	// remove this change.
+	// See https://issues.redhat.com//browse/OCPBUGS-32510 for details.
+	if !f.infrastructure.HighlyAvailableInfrastructure() {
+		containers[idx].StartupProbe = containers[idx].ReadinessProbe.DeepCopy()
+		containers[idx].ReadinessProbe.HTTPGet.Path = "/livez"
+	}
+
 	// Hash the Kubelet Serving CA Bundle configmap value and propagate it as a annotation to the
 	// deployment's pods to trigger a new rollout when the CA is rotated.
 	dep.Spec.Template.Annotations["monitoring.openshift.io/kubelet-serving-ca-bundle-hash"] = hashStringMap(kubeletCABundle.Data)

pkg/manifests/manifests_test.go

@@ -2856,6 +2856,75 @@ metricsServer:
 	require.Equal(t, "383c7cmidrae2", podAnnotations["monitoring.openshift.io/serving-ca-secret-hash"])
 }
 
+func TestMetricsServerReadinessProbe(t *testing.T) {
+	c, err := NewConfigFromString("", true)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c.SetImages(map[string]string{
+		"kube-metrics-server": "docker.io/openshift/origin-kube-metrics-server:latest",
+	})
+
+	f := NewFactory("openshift-monitoring", "openshift-user-workload-monitoring", c, &fakeInfrastructureReader{}, &fakeProxyReader{}, NewAssets(assetsPath), &APIServerConfig{}, &configv1.Console{})
+	kubeletCABundle := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "kubelet-serving-ca-bundle",
+			Namespace: "openshift-monitoring",
+		},
+		Data: map[string]string{
+			"ca-bundle.crt": "ca-certificate",
+		},
+	}
+	servingCASecret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "metrics-server-tls",
+			Namespace: "openshift-monitoring",
+		},
+		Data: map[string][]byte{
+			"tls.crt": []byte("foo"),
+			"tls.key": []byte("bar"),
+		},
+	}
+	metricsClientSecret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "metrics-client-cert",
+			Namespace: "openshift-monitoring",
+		},
+		Data: map[string][]byte{
+			"tls.crt": []byte("bar"),
+			"tls.key": []byte("foo"),
+		},
+	}
+	apiAuthConfigMapData := map[string]string{
+		"requestheader-allowed-names":        "",
+		"requestheader-extra-headers-prefix": "",
+		"requestheader-group-headers":        "",
+		"requestheader-username-headers":     "",
+	}
+
+	d, err := f.MetricsServerDeployment("foo", kubeletCABundle, servingCASecret, metricsClientSecret, apiAuthConfigMapData)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, container := range d.Spec.Template.Spec.Containers {
+		if container.Name == "metrics-server" {
+			if container.ReadinessProbe.HTTPGet.Path != "/livez" {
+				t.Fatalf("expected readiness probe's path to be '/livez', got %q", container.ReadinessProbe.HTTPGet.Path)
+			}
+
+			if container.StartupProbe.HTTPGet.Path != "/readyz" {
+				t.Fatalf("expected startup probe's path to be '/readyz', got %q", container.StartupProbe.HTTPGet.Path)
+			}
+
+			return
+		}
+	}
+
+	t.Fatalf("failed to find container %q", "metrics-server")
+}
+
 func TestMetricsServerAuditLog(t *testing.T) {
 	argsForProfile := func(profile string) []string {
 		return []string{