test(e2e/MeshCircuitBreaker): add tests for delegated gateway (#8832)

Add e2e tests of MeshCircuitBreaker targetting delegated gateways

Signed-off-by: Bart Smykla <[email protected]>

test/e2e_env/kubernetes/gateway/delegated.go

@@ -0,0 +1,87 @@
+package gateway
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	. "github.com/kumahq/kuma/test/framework"
+	"github.com/kumahq/kuma/test/framework/deployments/democlient"
+	"github.com/kumahq/kuma/test/framework/deployments/kic"
+	"github.com/kumahq/kuma/test/framework/deployments/testserver"
+	"github.com/kumahq/kuma/test/framework/envs/kubernetes"
+)
+
+type delegatedE2EConfig struct {
+	namespace            string
+	namespaceOutsideMesh string
+	mesh                 string
+	kicIP                string
+}
+
+func Delegated() {
+	config := &delegatedE2EConfig{
+		namespace:            "delegated-gateway",
+		namespaceOutsideMesh: "delegated-gateway-outside-mesh",
+		mesh:                 "delegated-gateway",
+		kicIP:                "",
+	}
+
+	BeforeAll(func() {
+		err := NewClusterSetup().
+			Install(MTLSMeshKubernetes(config.mesh)).
+			Install(NamespaceWithSidecarInjection(config.namespace)).
+			Install(Namespace(config.namespaceOutsideMesh)).
+			Install(democlient.Install(
+				democlient.WithNamespace(config.namespaceOutsideMesh),
+			)).
+			Install(testserver.Install(
+				testserver.WithMesh(config.mesh),
+				testserver.WithNamespace(config.namespace),
+				testserver.WithName("test-server"),
+			)).
+			Install(kic.KongIngressController(
+				kic.WithNamespace(config.namespace),
+				kic.WithMesh(config.mesh),
+			)).
+			Install(kic.KongIngressService(kic.WithNamespace(config.namespace))).
+			Install(YamlK8s(fmt.Sprintf(`
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  namespace: %s
+  name: %s-ingress
+  annotations:
+    kubernetes.io/ingress.class: kong
+spec:
+  rules:
+  - http:
+      paths:
+      - path: /test-server
+        pathType: Prefix
+        backend:
+          service:
+            name: test-server
+            port:
+              number: 80
+`, config.namespace, config.mesh))).
+			Setup(kubernetes.Cluster)
+		Expect(err).ToNot(HaveOccurred())
+
+		kicIP, err := kic.From(kubernetes.Cluster).IP(config.namespace)
+		Expect(err).To(Succeed())
+
+		config.kicIP = kicIP
+	})
+
+	E2EAfterAll(func() {
+		Expect(kubernetes.Cluster.TriggerDeleteNamespace(config.namespace)).
+			To(Succeed())
+		Expect(kubernetes.Cluster.TriggerDeleteNamespace(config.namespaceOutsideMesh)).
+			To(Succeed())
+		Expect(kubernetes.Cluster.DeleteMesh(config.mesh)).To(Succeed())
+	})
+
+	Context("MeshCircuitBreaker", CircuitBreaker(config))
+}

test/e2e_env/kubernetes/gateway/delegated_meshcircuitbreaker.go

@@ -0,0 +1,126 @@
+package gateway
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
+	"github.com/kumahq/kuma/pkg/plugins/policies/meshcircuitbreaker/api/v1alpha1"
+	. "github.com/kumahq/kuma/test/framework"
+	"github.com/kumahq/kuma/test/framework/client"
+	"github.com/kumahq/kuma/test/framework/envs/kubernetes"
+)
+
+func CircuitBreaker(config *delegatedE2EConfig) func() {
+	GinkgoHelper()
+
+	return func() {
+		BeforeAll(func() {
+			Expect(DeleteMeshResources(
+				kubernetes.Cluster,
+				config.mesh,
+				core_mesh.CircuitBreakerResourceTypeDescriptor,
+				core_mesh.RetryResourceTypeDescriptor,
+				v1alpha1.MeshCircuitBreakerResourceTypeDescriptor,
+			)).To(Succeed())
+		})
+
+		E2EAfterEach(func() {
+			Expect(DeleteMeshResources(
+				kubernetes.Cluster,
+				config.mesh,
+				v1alpha1.MeshCircuitBreakerResourceTypeDescriptor,
+			)).To(Succeed())
+		})
+
+		DescribeTable("should configure circuit breaker limits and outlier"+
+			" detectors for connections", func(yaml string) {
+			// given no MeshCircuitBreaker
+			mcbs, err := kubernetes.Cluster.GetKumactlOptions().
+				KumactlList("meshcircuitbreakers", config.mesh)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(mcbs).To(BeEmpty())
+
+			Eventually(func() ([]client.FailureResponse, error) {
+				return client.CollectResponsesAndFailures(
+					kubernetes.Cluster,
+					"demo-client",
+					fmt.Sprintf("http://%s/test-server", config.kicIP),
+					client.FromKubernetesPod(config.namespaceOutsideMesh, "demo-client"),
+					client.WithNumberOfRequests(10),
+				)
+			}, "30s", "1s").Should(And(
+				HaveLen(10),
+				HaveEach(HaveField("ResponseCode", 200)),
+			))
+
+			// when
+			Expect(kubernetes.Cluster.Install(YamlK8s(yaml))).To(Succeed())
+
+			// then
+			Eventually(func(g Gomega) ([]client.FailureResponse, error) {
+				return client.CollectResponsesAndFailures(
+					kubernetes.Cluster,
+					"demo-client",
+					fmt.Sprintf("http://%s/test-server", config.kicIP),
+					client.FromKubernetesPod(config.namespaceOutsideMesh, "demo-client"),
+					client.WithNumberOfRequests(10),
+					// increase processing time of a request to increase
+					// a probability of triggering maxPendingRequest limit
+					client.WithHeader("x-set-response-delay-ms", "1000"),
+					client.WithoutRetries(),
+				)
+			}, "30s", "1s").Should(And(
+				HaveLen(10),
+				ContainElement(HaveField("ResponseCode", 503)),
+			))
+		},
+			Entry("outbound circuit breaker", fmt.Sprintf(`
+apiVersion: kuma.io/v1alpha1
+kind: MeshCircuitBreaker
+metadata:
+  name: mcb-outbound
+  namespace: %s
+  labels:
+    kuma.io/mesh: %s
+spec:
+  targetRef:
+    kind: Mesh
+  to:
+    - targetRef:
+        kind: Mesh
+      default:
+        connectionLimits:
+          maxConnectionPools: 1
+          maxConnections: 1
+          maxPendingRequests: 1
+          maxRequests: 1
+          maxRetries: 1
+`, Config.KumaNamespace, config.mesh)),
+			Entry("inbound circuit breaker", fmt.Sprintf(`
+apiVersion: kuma.io/v1alpha1
+kind: MeshCircuitBreaker
+metadata:
+  name: mcb-inbound
+  namespace: %s
+  labels:
+    kuma.io/mesh: %s
+spec:
+  targetRef:
+    kind: Mesh
+  from:
+    - targetRef:
+        kind: Mesh
+      default:
+        connectionLimits:
+          maxConnectionPools: 1
+          maxConnections: 1
+          maxPendingRequests: 1
+          maxRequests: 1
+          maxRetries: 1
+`, Config.KumaNamespace, config.mesh)),
+		)
+	}
+}

test/e2e_env/kubernetes/kubernetes_suite_test.go

@@ -56,6 +56,7 @@ var (
 	_ = Describe("Gateway - Gateway API", gateway.GatewayAPI, Ordered)
 	_ = Describe("Gateway - mTLS", gateway.Mtls, Ordered)
 	_ = Describe("Gateway - Resources", gateway.Resources, Ordered)
+	_ = Describe("Delegated Gateway", Label("kind-not-supported", "ipv6-not-supported", "arm-not-supported"), gateway.Delegated, Ordered)
 	_ = Describe("Graceful", graceful.Graceful, Ordered)
 	_ = Describe("Eviction", graceful.Eviction, Ordered)
 	_ = XDescribe("Change Service", graceful.ChangeService, Ordered)