test/k8s: replace L7 visibility Pod annotations by L7 visibility policy

[ upstream commit 7793a07 ]

L7 visibility using Pod annotations is deprecated and the feature will
be removed in a successive commit. Switch K8sAgentHubbleTest still using
it to use L7 visibility policy instead.

Signed-off-by: Tobias Klauser <[email protected]>

Author: tklauser

test/k8s/hubble.go

@@ -8,13 +8,11 @@ import (
 	"fmt"
 	"net"
 	"strconv"
-	"strings"
 
 	. "github.com/onsi/gomega"
 	"google.golang.org/protobuf/encoding/protojson"
 
 	observerpb "github.com/cilium/cilium/api/v1/observer"
-	"github.com/cilium/cilium/pkg/annotation"
 	"github.com/cilium/cilium/pkg/hubble/defaults"
 	"github.com/cilium/cilium/pkg/identity"
 	. "github.com/cilium/cilium/test/ginkgo-ext"
@@ -38,7 +36,8 @@ var _ = Describe("K8sAgentHubbleTest", func() {
 			hubbleRelayService   = "hubble-relay"
 			hubbleRelayAddress   string
 
-			demoPath string
+			demoPath             string
+			visibilityPolicyPath string
 
 			app1Service    = "app1-service"
 			app1Labels     = "id=app1,zgroup=testapp"
@@ -51,37 +50,15 @@ var _ = Describe("K8sAgentHubbleTest", func() {
 			app1Port         int
 		)
 
-		addVisibilityAnnotation := func(ns, podLabels, direction, port, l4proto, l7proto string) {
-			visibilityAnnotation := fmt.Sprintf("<%s/%s/%s/%s>", direction, port, l4proto, l7proto)
-			By("Adding visibility annotation %s on pod with labels %s", visibilityAnnotation, podLabels)
-
-			// Prints <node>=<ns>/<podname> for each pod the annotation was applied to
-			res := kubectl.Exec(fmt.Sprintf("%s annotate pod -n %s -l %s %s=%q"+
-				" -o 'jsonpath={.spec.nodeName}={.metadata.namespace}/{.metadata.name}{\"\\n\"}'",
-				helpers.KubectlCmd,
-				ns, app1Labels,
-				annotation.ProxyVisibility, visibilityAnnotation))
-			res.ExpectSuccess("adding proxy visibility annotation failed")
-
-			// For each pod, check that the Cilium proxy-statistics contain the new annotation
-			expectedProxyState := strings.ToLower(visibilityAnnotation)
-			for node, podName := range res.KVOutput() {
-				ciliumPod, err := kubectl.GetCiliumPodOnNodeByName(node)
-				Expect(err).To(BeNil())
-
-				// Extract annotation from endpoint model of pod. It does not have the l4proto, so we insert it manually.
-				cmd := fmt.Sprintf("cilium-dbg endpoint get pod-name:%s"+
-					" -o jsonpath='{range [*].status.policy.proxy-statistics[*]}<{.location}/{.port}/%s/{.protocol}>{\"\\n\"}{end}'",
-					podName, strings.ToLower(l4proto))
-				err = kubectl.CiliumExecUntilMatch(ciliumPod, cmd, expectedProxyState)
-				Expect(err).To(BeNil(), "timed out waiting for endpoint to regenerate for visibility annotation")
-			}
+		addVisibilityPolicy := func(ns string) {
+			By("Applying L7 visibility policy")
+			res := kubectl.Apply(helpers.ApplyOptions{FilePath: visibilityPolicyPath, Namespace: ns})
+			res.ExpectSuccess("could not create L7 visibility policy")
 		}
 
-		removeVisibilityAnnotation := func(ns, podLabels string) {
-			By("Removing visibility annotation on pod with labels %s", app1Labels)
-			res := kubectl.Exec(fmt.Sprintf("%s annotate pod -n %s -l %s %s-", helpers.KubectlCmd, ns, podLabels, annotation.ProxyVisibility))
-			res.ExpectSuccess("removing proxy visibility annotation failed")
+		removeVisibilityPolicy := func(ns string) {
+			By("Removing L7 visibility policy")
+			kubectl.DeleteInNamespace(ns, visibilityPolicyPath)
 		}
 
 		getFlowsFromRelay := func(args string) []*observerpb.GetFlowsResponse {
@@ -123,6 +100,7 @@ var _ = Describe("K8sAgentHubbleTest", func() {
 			k8s1NodeName, _ = kubectl.GetNodeInfo(helpers.K8s1)
 
 			demoPath = helpers.ManifestGet(kubectl.BasePath(), "demo.yaml")
+			visibilityPolicyPath = helpers.ManifestGet(kubectl.BasePath(), "l7-policy-visibility.yaml")
 
 			ciliumFilename = helpers.TimestampFilename("cilium.yaml")
 			DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{
@@ -225,8 +203,8 @@ var _ = Describe("K8sAgentHubbleTest", func() {
 		})
 
 		It("Test L7 Flow", func() {
-			defer removeVisibilityAnnotation(namespaceForTest, app1Labels)
-			addVisibilityAnnotation(namespaceForTest, app1Labels, "Ingress", "80", "TCP", "HTTP")
+			defer removeVisibilityPolicy(namespaceForTest)
+			addVisibilityPolicy(namespaceForTest)
 
 			ctx, cancel := context.WithTimeout(context.Background(), helpers.MidCommandTimeout)
 			defer cancel()
@@ -251,8 +229,8 @@ var _ = Describe("K8sAgentHubbleTest", func() {
 		})
 
 		It("Test L7 Flow with hubble-relay", func() {
-			defer removeVisibilityAnnotation(namespaceForTest, app1Labels)
-			addVisibilityAnnotation(namespaceForTest, app1Labels, "Ingress", "80", "TCP", "HTTP")
+			defer removeVisibilityPolicy(namespaceForTest)
+			addVisibilityPolicy(namespaceForTest)
 
 			res := kubectl.ExecPodCmd(namespaceForTest, appPods[helpers.App2],
 				helpers.CurlFail(fmt.Sprintf("http://%s/public", app1ClusterIP)))

test/k8s/manifests/l7-policy-visibility.yaml

@@ -0,0 +1,16 @@
+apiVersion: "cilium.io/v2"
+kind: CiliumNetworkPolicy
+metadata:
+  name: "l7-visibility"
+spec:
+  endpointSelector:
+    matchLabels:
+      id: app1
+      zgroup: testapp
+  ingress:
+  - toPorts:
+    - ports:
+      - port: "80"
+        protocol: TCP
+      rules:
+        http: [{}]