Skip to content

Commit b703a44

Browse files
mayuka-cmayuka-c
committed
[e2e] Fixing flakes in multigroup serve metrics
Fix flakes - 2 Run make generate RUn make generate Minor fix Minor fix Fix-2 Fix-3 Implement a new marker for webhook readiness
1 parent 17fc377 commit b703a44

File tree

8 files changed

+299
-16
lines changed

8 files changed

+299
-16
lines changed

‎docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,50 @@ var _ = Describe("Manager", Ordered, func() {
216216
}
217217
Eventually(verifyMetricsServerStarted).Should(Succeed())
218218

219+
By("waiting for webhook service to be ready if webhooks are configured")
220+
verifyWebhookServiceReady := func(g Gomega) {
221+
// Try to find webhook service - use a more generic approach to handle custom configurations
222+
cmd := exec.Command("kubectl", "get", "services", "-n", namespace,
223+
"-l", "app.kubernetes.io/component=webhook",
224+
"-o", "jsonpath={.items[0].metadata.name}")
225+
webhookServiceName, err := utils.Run(cmd)
226+
if err != nil || webhookServiceName == "" {
227+
// Fallback to default naming convention
228+
webhookServiceName = "project-webhook-service"
229+
cmd = exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
230+
_, err = utils.Run(cmd)
231+
if err != nil {
232+
// No webhook service found, skip webhook checks
233+
return
234+
}
235+
}
236+
237+
// Check if webhook server is ready by verifying pod readiness
238+
cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
239+
"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
240+
output, err := utils.Run(cmd)
241+
g.Expect(err).NotTo(HaveOccurred())
242+
g.Expect(output).To(Equal("True"),
243+
"Controller manager pod not ready (webhook server may not be accepting connections)")
244+
245+
// Check if webhook service endpoints are available
246+
cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
247+
"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
248+
output, err = utils.Run(cmd)
249+
g.Expect(err).NotTo(HaveOccurred())
250+
g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")
251+
252+
// Test webhook connectivity by checking if webhook server port is responding
253+
cmd = exec.Command("kubectl", "run", "webhook-test", "--rm", "-i", "--restart=Never",
254+
"--image=curlimages/curl:latest", "--",
255+
"curl", "-k", "--connect-timeout", "5",
256+
"https://"+webhookServiceName+"."+namespace+".svc:443/readyz")
257+
_, err = utils.Run(cmd)
258+
g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
259+
}
260+
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())
261+
// +kubebuilder:scaffold:e2e-webhooks-readiness
262+
219263
By("creating the curl-metrics pod to access the metrics endpoint")
220264
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
221265
"--namespace", namespace,

‎docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,8 @@ var _ = Describe("Manager", Ordered, func() {
211211
}
212212
Eventually(verifyMetricsServerStarted).Should(Succeed())
213213

214+
// +kubebuilder:scaffold:e2e-webhooks-readiness
215+
214216
By("creating the curl-metrics pod to access the metrics endpoint")
215217
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
216218
"--namespace", namespace,

‎docs/book/src/multiversion-tutorial/testdata/project/test/e2e/e2e_test.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,50 @@ var _ = Describe("Manager", Ordered, func() {
216216
}
217217
Eventually(verifyMetricsServerStarted).Should(Succeed())
218218

219+
By("waiting for webhook service to be ready if webhooks are configured")
220+
verifyWebhookServiceReady := func(g Gomega) {
221+
// Try to find webhook service - use a more generic approach to handle custom configurations
222+
cmd := exec.Command("kubectl", "get", "services", "-n", namespace,
223+
"-l", "app.kubernetes.io/component=webhook",
224+
"-o", "jsonpath={.items[0].metadata.name}")
225+
webhookServiceName, err := utils.Run(cmd)
226+
if err != nil || webhookServiceName == "" {
227+
// Fallback to default naming convention
228+
webhookServiceName = "project-webhook-service"
229+
cmd = exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
230+
_, err = utils.Run(cmd)
231+
if err != nil {
232+
// No webhook service found, skip webhook checks
233+
return
234+
}
235+
}
236+
237+
// Check if webhook server is ready by verifying pod readiness
238+
cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
239+
"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
240+
output, err := utils.Run(cmd)
241+
g.Expect(err).NotTo(HaveOccurred())
242+
g.Expect(output).To(Equal("True"),
243+
"Controller manager pod not ready (webhook server may not be accepting connections)")
244+
245+
// Check if webhook service endpoints are available
246+
cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
247+
"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
248+
output, err = utils.Run(cmd)
249+
g.Expect(err).NotTo(HaveOccurred())
250+
g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")
251+
252+
// Test webhook connectivity by checking if webhook server port is responding
253+
cmd = exec.Command("kubectl", "run", "webhook-test", "--rm", "-i", "--restart=Never",
254+
"--image=curlimages/curl:latest", "--",
255+
"curl", "-k", "--connect-timeout", "5",
256+
"https://"+webhookServiceName+"."+namespace+".svc:443/readyz")
257+
_, err = utils.Run(cmd)
258+
g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
259+
}
260+
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())
261+
// +kubebuilder:scaffold:e2e-webhooks-readiness
262+
219263
By("creating the curl-metrics pod to access the metrics endpoint")
220264
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
221265
"--namespace", namespace,

‎docs/book/src/reference/markers/scaffold.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ properly registered with the manager, so that the controller can reconcile the r
107107
| `+kubebuilder:scaffold:crdkustomizecainjectioname` | `config/default` | Marks where CA injection patches are added for the conversion webhooks. |
108108
| **(No longer supported)** `+kubebuilder:scaffold:crdkustomizecainjectionpatch` | `config/crd` | Marks where CA injection patches are added for the webhooks. Replaced by `+kubebuilder:scaffold:crdkustomizecainjectionns` and `+kubebuilder:scaffold:crdkustomizecainjectioname` |
109109
| `+kubebuilder:scaffold:manifestskustomizesamples` | `config/samples` | Marks where Kustomize sample manifests are injected. |
110+
| `+kubebuilder:scaffold:e2e-webhook-readiness` | `test/e2e` | Adds webhook readiness checks in e2e tests before metrics collection. |
110111
| `+kubebuilder:scaffold:e2e-webhooks-checks` | `test/e2e` | Adds e2e checks for webhooks depending on the types of webhooks scaffolded. |
111112

112113
<aside class="note warning">

‎pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go‎

Lines changed: 76 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
log "log/slog"
2323
"os"
2424
"path/filepath"
25+
"strings"
2526

2627
"sigs.k8s.io/kubebuilder/v4/pkg/machinery"
2728
)
@@ -32,6 +33,7 @@ var (
3233
)
3334

3435
const webhookChecksMarker = "e2e-webhooks-checks"
36+
const webhookReadinessMarker = "e2e-webhooks-readiness"
3537

3638
// Test defines the basic setup for the e2e test
3739
type Test struct {
@@ -74,6 +76,7 @@ func (*WebhookTestUpdater) GetIfExistsAction() machinery.IfExistsAction {
7476
// GetMarkers implements file.Inserter
7577
func (f *WebhookTestUpdater) GetMarkers() []machinery.Marker {
7678
return []machinery.Marker{
79+
machinery.NewMarkerFor(f.GetPath(), webhookReadinessMarker),
7780
machinery.NewMarkerFor(f.GetPath(), webhookChecksMarker),
7881
}
7982
}
@@ -107,25 +110,36 @@ func (f *WebhookTestUpdater) GetCodeFragments() machinery.CodeFragmentsMap {
107110
}
108111

109112
var fragments []string
110-
fragments = append(fragments, webhookChecksFragment)
111113

112-
if f.Resource != nil && f.Resource.HasDefaultingWebhook() {
113-
mutatingWebhookCode := fmt.Sprintf(mutatingWebhookChecksFragment, f.ProjectName)
114-
fragments = append(fragments, mutatingWebhookCode)
115-
}
114+
// Handle different markers differently
115+
markerString := marker.String()
116+
if strings.Contains(markerString, webhookReadinessMarker) {
117+
// Only inject readiness check for this marker
118+
webhookReadinessCode := fmt.Sprintf(webhookReadinessFragment, f.ProjectName)
119+
fragments = append(fragments, webhookReadinessCode)
116120

117-
if f.Resource != nil && f.Resource.HasValidationWebhook() {
118-
validatingWebhookCode := fmt.Sprintf(validatingWebhookChecksFragment, f.ProjectName)
119-
fragments = append(fragments, validatingWebhookCode)
120-
}
121+
} else if strings.Contains(markerString, webhookChecksMarker) {
122+
// Inject all other webhook tests for this marker
123+
fragments = append(fragments, webhookChecksFragment)
121124

122-
if f.Resource != nil && f.Resource.HasConversionWebhook() {
123-
conversionWebhookCode := fmt.Sprintf(
124-
conversionWebhookChecksFragment,
125-
f.Resource.Kind,
126-
f.Resource.Plural+"."+f.Resource.Group+"."+f.Resource.Domain,
127-
)
128-
fragments = append(fragments, conversionWebhookCode)
125+
if f.Resource != nil && f.Resource.HasDefaultingWebhook() {
126+
mutatingWebhookCode := fmt.Sprintf(mutatingWebhookChecksFragment, f.ProjectName)
127+
fragments = append(fragments, mutatingWebhookCode)
128+
}
129+
130+
if f.Resource != nil && f.Resource.HasValidationWebhook() {
131+
validatingWebhookCode := fmt.Sprintf(validatingWebhookChecksFragment, f.ProjectName)
132+
fragments = append(fragments, validatingWebhookCode)
133+
}
134+
135+
if f.Resource != nil && f.Resource.HasConversionWebhook() {
136+
conversionWebhookCode := fmt.Sprintf(
137+
conversionWebhookChecksFragment,
138+
f.Resource.Kind,
139+
f.Resource.Plural+"."+f.Resource.Group+"."+f.Resource.Domain,
140+
)
141+
fragments = append(fragments, conversionWebhookCode)
142+
}
129143
}
130144

131145
codeFragments[marker] = fragments
@@ -198,6 +212,50 @@ const conversionWebhookChecksFragment = `It("should have CA injection for %[1]s
198212
199213
`
200214

215+
const webhookReadinessFragment = ` By("waiting for webhook service to be ready if webhooks are configured")
216+
verifyWebhookServiceReady := func(g Gomega) {
217+
// Try to find webhook service - use a more generic approach to handle custom configurations
218+
cmd := exec.Command("kubectl", "get", "services", "-n", namespace,
219+
"-l", "app.kubernetes.io/component=webhook",
220+
"-o", "jsonpath={.items[0].metadata.name}")
221+
webhookServiceName, err := utils.Run(cmd)
222+
if err != nil || webhookServiceName == "" {
223+
// Fallback to default naming convention
224+
webhookServiceName = "%s-webhook-service"
225+
cmd = exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
226+
_, err = utils.Run(cmd)
227+
if err != nil {
228+
// No webhook service found, skip webhook checks
229+
return
230+
}
231+
}
232+
233+
// Check if webhook server is ready by verifying pod readiness
234+
cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
235+
"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
236+
output, err := utils.Run(cmd)
237+
g.Expect(err).NotTo(HaveOccurred())
238+
g.Expect(output).To(Equal("True"),
239+
"Controller manager pod not ready (webhook server may not be accepting connections)")
240+
241+
// Check if webhook service endpoints are available
242+
cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
243+
"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
244+
output, err = utils.Run(cmd)
245+
g.Expect(err).NotTo(HaveOccurred())
246+
g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")
247+
248+
// Test webhook connectivity by checking if webhook server port is responding
249+
cmd = exec.Command("kubectl", "run", "webhook-test", "--rm", "-i", "--restart=Never",
250+
"--image=curlimages/curl:latest", "--",
251+
"curl", "-k", "--connect-timeout", "5",
252+
"https://"+webhookServiceName+"."+namespace+".svc:443/readyz")
253+
_, err = utils.Run(cmd)
254+
g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
255+
}
256+
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())
257+
`
258+
201259
var testCodeTemplate = `//go:build e2e
202260
// +build e2e
203261
@@ -394,6 +452,8 @@ var _ = Describe("Manager", Ordered, func() {
394452
}
395453
Eventually(verifyMetricsServerStarted).Should(Succeed())
396454
455+
// +kubebuilder:scaffold:e2e-webhooks-readiness
456+
397457
By("creating the curl-metrics pod to access the metrics endpoint")
398458
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
399459
"--namespace", namespace,

‎testdata/project-v4-multigroup/test/e2e/e2e_test.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,50 @@ var _ = Describe("Manager", Ordered, func() {
211211
}
212212
Eventually(verifyMetricsServerStarted).Should(Succeed())
213213

214+
By("waiting for webhook service to be ready if webhooks are configured")
215+
verifyWebhookServiceReady := func(g Gomega) {
216+
// Try to find webhook service - use a more generic approach to handle custom configurations
217+
cmd := exec.Command("kubectl", "get", "services", "-n", namespace,
218+
"-l", "app.kubernetes.io/component=webhook",
219+
"-o", "jsonpath={.items[0].metadata.name}")
220+
webhookServiceName, err := utils.Run(cmd)
221+
if err != nil || webhookServiceName == "" {
222+
// Fallback to default naming convention
223+
webhookServiceName = "project-v4-multigroup-webhook-service"
224+
cmd = exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
225+
_, err = utils.Run(cmd)
226+
if err != nil {
227+
// No webhook service found, skip webhook checks
228+
return
229+
}
230+
}
231+
232+
// Check if webhook server is ready by verifying pod readiness
233+
cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
234+
"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
235+
output, err := utils.Run(cmd)
236+
g.Expect(err).NotTo(HaveOccurred())
237+
g.Expect(output).To(Equal("True"),
238+
"Controller manager pod not ready (webhook server may not be accepting connections)")
239+
240+
// Check if webhook service endpoints are available
241+
cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
242+
"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
243+
output, err = utils.Run(cmd)
244+
g.Expect(err).NotTo(HaveOccurred())
245+
g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")
246+
247+
// Test webhook connectivity by checking if webhook server port is responding
248+
cmd = exec.Command("kubectl", "run", "webhook-test", "--rm", "-i", "--restart=Never",
249+
"--image=curlimages/curl:latest", "--",
250+
"curl", "-k", "--connect-timeout", "5",
251+
"https://"+webhookServiceName+"."+namespace+".svc:443/readyz")
252+
_, err = utils.Run(cmd)
253+
g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
254+
}
255+
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())
256+
// +kubebuilder:scaffold:e2e-webhooks-readiness
257+
214258
By("creating the curl-metrics pod to access the metrics endpoint")
215259
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
216260
"--namespace", namespace,

‎testdata/project-v4-with-plugins/test/e2e/e2e_test.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,50 @@ var _ = Describe("Manager", Ordered, func() {
211211
}
212212
Eventually(verifyMetricsServerStarted).Should(Succeed())
213213

214+
By("waiting for webhook service to be ready if webhooks are configured")
215+
verifyWebhookServiceReady := func(g Gomega) {
216+
// Try to find webhook service - use a more generic approach to handle custom configurations
217+
cmd := exec.Command("kubectl", "get", "services", "-n", namespace,
218+
"-l", "app.kubernetes.io/component=webhook",
219+
"-o", "jsonpath={.items[0].metadata.name}")
220+
webhookServiceName, err := utils.Run(cmd)
221+
if err != nil || webhookServiceName == "" {
222+
// Fallback to default naming convention
223+
webhookServiceName = "project-v4-with-plugins-webhook-service"
224+
cmd = exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
225+
_, err = utils.Run(cmd)
226+
if err != nil {
227+
// No webhook service found, skip webhook checks
228+
return
229+
}
230+
}
231+
232+
// Check if webhook server is ready by verifying pod readiness
233+
cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
234+
"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
235+
output, err := utils.Run(cmd)
236+
g.Expect(err).NotTo(HaveOccurred())
237+
g.Expect(output).To(Equal("True"),
238+
"Controller manager pod not ready (webhook server may not be accepting connections)")
239+
240+
// Check if webhook service endpoints are available
241+
cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
242+
"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
243+
output, err = utils.Run(cmd)
244+
g.Expect(err).NotTo(HaveOccurred())
245+
g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")
246+
247+
// Test webhook connectivity by checking if webhook server port is responding
248+
cmd = exec.Command("kubectl", "run", "webhook-test", "--rm", "-i", "--restart=Never",
249+
"--image=curlimages/curl:latest", "--",
250+
"curl", "-k", "--connect-timeout", "5",
251+
"https://"+webhookServiceName+"."+namespace+".svc:443/readyz")
252+
_, err = utils.Run(cmd)
253+
g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
254+
}
255+
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())
256+
// +kubebuilder:scaffold:e2e-webhooks-readiness
257+
214258
By("creating the curl-metrics pod to access the metrics endpoint")
215259
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
216260
"--namespace", namespace,

0 commit comments

Comments
 (0)