envoy: Configure internal address config based on IP family

[upstream d9e64fc]

There is upstream issue which might cause IP malformed error, this
commit is to make sure that ::1/128 is added to internal address config
if IPv6 is enabled in Cilium.

Relates: envoyproxy/envoy#29902
Fixes: cilium#35943
Fixes: cilium#36691
Signed-off-by: Tam Mach <[email protected]>

install/kubernetes/cilium/files/cilium-envoy/configmap/bootstrap-config.yaml

@@ -33,6 +33,7 @@ staticResources:
               "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
           internalAddressConfig:
             cidrRanges:
+            {{- if .Values.ipv4.enabled }}
             - addressPrefix: "10.0.0.0"
               prefixLen: 8
             - addressPrefix: "172.16.0.0"
@@ -41,8 +42,11 @@ staticResources:
               prefixLen: 16
             - addressPrefix: "127.0.0.1"
               prefixLen: 32
+            {{- end }}
+            {{- if .Values.ipv6.enabled }}
             - addressPrefix: "::1"
               prefixLen: 128
+            {{- end }}
           streamIdleTimeout: "0s"
   {{- end }}
   {{- if and .Values.envoy.debug.admin.enabled }}
@@ -82,6 +86,7 @@ staticResources:
               "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
           internalAddressConfig:
             cidrRanges:
+            {{- if .Values.ipv4.enabled }}
             - addressPrefix: "10.0.0.0"
               prefixLen: 8
             - addressPrefix: "172.16.0.0"
@@ -90,8 +95,11 @@ staticResources:
               prefixLen: 16
             - addressPrefix: "127.0.0.1"
               prefixLen: 32
+            {{- end }}
+            {{- if .Values.ipv6.enabled }}
             - addressPrefix: "::1"
               prefixLen: 128
+            {{- end }}
           streamIdleTimeout: "0s"
   {{- end }}
   - name: "envoy-health-listener"
@@ -130,6 +138,7 @@ staticResources:
               "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
           internalAddressConfig:
             cidrRanges:
+            {{- if .Values.ipv4.enabled }}
             - addressPrefix: "10.0.0.0"
               prefixLen: 8
             - addressPrefix: "172.16.0.0"
@@ -138,8 +147,11 @@ staticResources:
               prefixLen: 16
             - addressPrefix: "127.0.0.1"
               prefixLen: 32
+            {{- end }}
+            {{- if .Values.ipv6.enabled }}
             - addressPrefix: "::1"
               prefixLen: 128
+            {{- end }}
           streamIdleTimeout: "0s"
   clusters:
   - name: "ingress-cluster"

operator/pkg/ciliumenvoyconfig/cell.go

@@ -12,6 +12,7 @@ import (
 	ctrlRuntime "sigs.k8s.io/controller-runtime"
 
 	operatorOption "github.com/cilium/cilium/operator/option"
+	agentOption "github.com/cilium/cilium/pkg/option"
 )
 
 // Cell manages the CiliumEnvoyConfig related controllers.
@@ -70,6 +71,8 @@ func registerL7LoadBalancingController(params l7LoadbalancerParams) error {
 		params.Config.LoadBalancerL7Ports,
 		10,
 		operatorOption.Config.ProxyIdleTimeoutSeconds,
+		agentOption.Config.EnableIPv4,
+		agentOption.Config.EnableIPv6,
 	)
 
 	if err := reconciler.SetupWithManager(params.CtrlRuntimeManager); err != nil {

operator/pkg/ciliumenvoyconfig/ciliumenvoyconfig.go

@@ -21,9 +21,12 @@ type ciliumEnvoyConfigReconciler struct {
 	ports              []string
 	maxRetries         int
 	idleTimeoutSeconds int
+	enableIpv4         bool
+	enableIpv6         bool
 }
 
-func newCiliumEnvoyConfigReconciler(c client.Client, logger logrus.FieldLogger, defaultAlgorithm string, ports []string, maxRetries int, idleTimeoutSeconds int) *ciliumEnvoyConfigReconciler {
+func newCiliumEnvoyConfigReconciler(c client.Client, logger logrus.FieldLogger, defaultAlgorithm string, ports []string,
+	maxRetries int, idleTimeoutSeconds int, enableIpv4 bool, enableIpv6 bool) *ciliumEnvoyConfigReconciler {
 	return &ciliumEnvoyConfigReconciler{
 		client: c,
 		logger: logger,
@@ -32,6 +35,8 @@ func newCiliumEnvoyConfigReconciler(c client.Client, logger logrus.FieldLogger,
 		ports:              ports,
 		maxRetries:         maxRetries,
 		idleTimeoutSeconds: idleTimeoutSeconds,
+		enableIpv4:         enableIpv4,
+		enableIpv6:         enableIpv6,
 	}
 }
 

operator/pkg/ciliumenvoyconfig/envoy_config.go

@@ -223,13 +223,7 @@ func (r *ciliumEnvoyConfigReconciler) getConnectionManager(svc *corev1.Service)
 			UnixSockets: false,
 			// only RFC1918 IP addresses will be considered internal
 			// https://datatracker.ietf.org/doc/html/rfc1918
-			CidrRanges: []*envoy_config_core_v3.CidrRange{
-				{AddressPrefix: "10.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: 8}},
-				{AddressPrefix: "172.16.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: 12}},
-				{AddressPrefix: "192.168.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: 16}},
-				{AddressPrefix: "127.0.0.1", PrefixLen: &wrapperspb.UInt32Value{Value: 32}},
-				{AddressPrefix: "::1", PrefixLen: &wrapperspb.UInt32Value{Value: 128}},
-			},
+			CidrRanges: envoy.GetInternalListenerCIDRs(r.enableIpv4, r.enableIpv6),
 		},
 	}
 

operator/pkg/model/translation/cec_translator.go

@@ -224,7 +224,8 @@ func (i *cecTranslator) getListener(m *model.Model) []ciliumv2.XDSResource {
 		mutatorFuncs = append(mutatorFuncs, WithXffNumTrustedHops(i.xffNumTrustedHops))
 	}
 
-	l, _ := newListenerWithDefaults("listener", i.secretsNamespace, len(m.HTTP) > 0, tlsSecretsToHostnames(m.HTTP), tlsPassthroughBackendsToHostnames(m.TLSPassthrough), mutatorFuncs...)
+	l, _ := newListenerWithDefaults("listener", i.secretsNamespace, len(m.HTTP) > 0, tlsSecretsToHostnames(m.HTTP),
+		tlsPassthroughBackendsToHostnames(m.TLSPassthrough), i.ipv4Enabled, i.ipv6Enabled, mutatorFuncs...)
 	return []ciliumv2.XDSResource{l}
 }
 

operator/pkg/model/translation/envoy_http_connection_manager.go

@@ -20,6 +20,16 @@ import (
 
 type HttpConnectionManagerMutator func(*httpConnectionManagerv3.HttpConnectionManager) *httpConnectionManagerv3.HttpConnectionManager
 
+func WithInternalAddressConfig(enableIpv4, enableIpv6 bool) HttpConnectionManagerMutator {
+	return func(hcm *httpConnectionManagerv3.HttpConnectionManager) *httpConnectionManagerv3.HttpConnectionManager {
+		hcm.InternalAddressConfig = &httpConnectionManagerv3.HttpConnectionManager_InternalAddressConfig{
+			UnixSockets: false,
+			CidrRanges:  envoy.GetInternalListenerCIDRs(enableIpv4, enableIpv6),
+		}
+		return hcm
+	}
+}
+
 // NewHTTPConnectionManager returns a new HTTP connection manager filter with the given name and route.
 // Mutation functions can be passed to modify the filter based on the caller's needs.
 func NewHTTPConnectionManager(name, routeName string, mutationFunc ...HttpConnectionManagerMutator) (ciliumv2.XDSResource, error) {
@@ -53,18 +63,6 @@ func NewHTTPConnectionManager(name, routeName string, mutationFunc ...HttpConnec
 				},
 			},
 		},
-		InternalAddressConfig: &httpConnectionManagerv3.HttpConnectionManager_InternalAddressConfig{
-			UnixSockets: false,
-			// only RFC1918 IP addresses will be considered internal
-			// https://datatracker.ietf.org/doc/html/rfc1918
-			CidrRanges: []*envoy_config_core.CidrRange{
-				{AddressPrefix: "10.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: 8}},
-				{AddressPrefix: "172.16.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: 12}},
-				{AddressPrefix: "192.168.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: 16}},
-				{AddressPrefix: "127.0.0.1", PrefixLen: &wrapperspb.UInt32Value{Value: 32}},
-				{AddressPrefix: "::1", PrefixLen: &wrapperspb.UInt32Value{Value: 128}},
-			},
-		},
 		UpgradeConfigs: []*httpConnectionManagerv3.HttpConnectionManager_UpgradeConfig{
 			{UpgradeType: "websocket"},
 		},

operator/pkg/model/translation/envoy_listener.go

@@ -176,7 +176,11 @@ func WithSocketOption(tcpKeepAlive, tcpKeepIdleInSeconds, tcpKeepAliveProbeInter
 }
 
 // newListenerWithDefaults same as newListener but with default mutators applied.
-func newListenerWithDefaults(name string, ciliumSecretNamespace string, includeHTTPFilterchain bool, tlsSecretsToHostnames map[model.TLSSecret][]string, ptBackendsToHostnames map[string][]string, mutatorFunc ...ListenerMutator) (ciliumv2.XDSResource, error) {
+func newListenerWithDefaults(name string, ciliumSecretNamespace string, includeHTTPFilterchain bool,
+	tlsSecretsToHostnames map[model.TLSSecret][]string,
+	ptBackendsToHostnames map[string][]string,
+	enableIpv4 bool, enableIpv6 bool,
+	mutatorFunc ...ListenerMutator) (ciliumv2.XDSResource, error) {
 	fns := append(mutatorFunc,
 		WithSocketOption(
 			defaultTCPKeepAlive,
@@ -185,14 +189,15 @@ func newListenerWithDefaults(name string, ciliumSecretNamespace string, includeH
 			defaultTCPKeepAliveMaxFailures),
 	)
 
-	return newListener(name, ciliumSecretNamespace, includeHTTPFilterchain, tlsSecretsToHostnames, ptBackendsToHostnames, fns...)
+	return newListener(name, ciliumSecretNamespace, includeHTTPFilterchain, tlsSecretsToHostnames, ptBackendsToHostnames, enableIpv4, enableIpv6, fns...)
 }
 
-func httpFilterChain(name string) (*envoy_config_listener.FilterChain, error) {
+func httpFilterChain(name string, enableIpv4 bool, enableIpv6 bool) (*envoy_config_listener.FilterChain, error) {
 	insecureHttpConnectionManagerName := fmt.Sprintf("%s-insecure", name)
 	insecureHttpConnectionManager, err := NewHTTPConnectionManager(
 		insecureHttpConnectionManagerName,
 		insecureHttpConnectionManagerName,
+		WithInternalAddressConfig(enableIpv4, enableIpv6),
 	)
 	if err != nil {
 		return nil, err
@@ -211,7 +216,8 @@ func httpFilterChain(name string) (*envoy_config_listener.FilterChain, error) {
 	}, nil
 }
 
-func httpsFilterChains(name string, ciliumSecretNamespace string, tlsSecretsToHostnames map[model.TLSSecret][]string) ([]*envoy_config_listener.FilterChain, error) {
+func httpsFilterChains(name string, ciliumSecretNamespace string, tlsSecretsToHostnames map[model.TLSSecret][]string,
+	enableIpv4 bool, enableIpv6 bool) ([]*envoy_config_listener.FilterChain, error) {
 	if len(tlsSecretsToHostnames) == 0 {
 		return nil, nil
 	}
@@ -225,7 +231,8 @@ func httpsFilterChains(name string, ciliumSecretNamespace string, tlsSecretsToHo
 		hostNames := tlsSecretsToHostnames[secret]
 
 		secureHttpConnectionManagerName := fmt.Sprintf("%s-secure", name)
-		secureHttpConnectionManager, err := NewHTTPConnectionManager(secureHttpConnectionManagerName, secureHttpConnectionManagerName)
+		secureHttpConnectionManager, err := NewHTTPConnectionManager(secureHttpConnectionManagerName, secureHttpConnectionManagerName,
+			WithInternalAddressConfig(enableIpv4, enableIpv6))
 		if err != nil {
 			return nil, err
 		}
@@ -256,18 +263,22 @@ func httpsFilterChains(name string, ciliumSecretNamespace string, tlsSecretsToHo
 // The listener will have both secure and insecure filters.
 //
 // Secret Discovery Service (SDS) is used to fetch the TLS certificates.
-func newListener(name string, ciliumSecretNamespace string, includeHTTPFilterchain bool, tlsSecretsToHostnames map[model.TLSSecret][]string, tlsPassthroughBackendsMap map[string][]string, mutatorFunc ...ListenerMutator) (ciliumv2.XDSResource, error) {
+func newListener(name string, ciliumSecretNamespace string, includeHTTPFilterchain bool,
+	tlsSecretsToHostnames map[model.TLSSecret][]string,
+	tlsPassthroughBackendsMap map[string][]string,
+	enableIpv4 bool, enableIpv6 bool,
+	mutatorFunc ...ListenerMutator) (ciliumv2.XDSResource, error) {
 	filterChains := []*envoy_config_listener.FilterChain{}
 
 	if includeHTTPFilterchain {
-		httpFilterChain, err := httpFilterChain(name)
+		httpFilterChain, err := httpFilterChain(name, enableIpv4, enableIpv6)
 		if err != nil {
 			return ciliumv2.XDSResource{}, err
 		}
 		filterChains = append(filterChains, httpFilterChain)
 	}
 
-	httpsFilterChains, err := httpsFilterChains(name, ciliumSecretNamespace, tlsSecretsToHostnames)
+	httpsFilterChains, err := httpsFilterChains(name, ciliumSecretNamespace, tlsSecretsToHostnames, enableIpv4, enableIpv6)
 	if err != nil {
 		return ciliumv2.XDSResource{}, fmt.Errorf("failed to create https filterchains: %w", err)
 	}

operator/pkg/model/translation/envoy_listener_test.go

@@ -23,7 +23,7 @@ import (
 
 func TestNewListener(t *testing.T) {
 	t.Run("Empty", func(t *testing.T) {
-		res, err := newListener("dummy-name", "dummy-secret-namespace", false, nil, nil)
+		res, err := newListener("dummy-name", "dummy-secret-namespace", false, nil, nil, true, true)
 		require.Nil(t, err)
 
 		listener := &envoy_config_listener.Listener{}
@@ -36,7 +36,7 @@ func TestNewListener(t *testing.T) {
 	})
 
 	t.Run("without TLS", func(t *testing.T) {
-		res, err := newListener("dummy-name", "dummy-secret-namespace", true, nil, nil)
+		res, err := newListener("dummy-name", "dummy-secret-namespace", true, nil, nil, true, true)
 		require.Nil(t, err)
 
 		listener := &envoy_config_listener.Listener{}
@@ -49,7 +49,7 @@ func TestNewListener(t *testing.T) {
 	})
 
 	t.Run("with default XffNumTrustedHops", func(t *testing.T) {
-		res, err := newListener("dummy-name", "dummy-secret-namespace", true, nil, nil)
+		res, err := newListener("dummy-name", "dummy-secret-namespace", true, nil, nil, true, true)
 		require.Nil(t, err)
 
 		listener := &envoy_config_listener.Listener{}
@@ -65,7 +65,7 @@ func TestNewListener(t *testing.T) {
 	})
 
 	t.Run("without TLS with Proxy Protocol", func(t *testing.T) {
-		res, err := newListener("dummy-name", "dummy-secret-namespace", true, nil, nil, WithProxyProtocol())
+		res, err := newListener("dummy-name", "dummy-secret-namespace", true, nil, nil, true, true, WithProxyProtocol())
 		require.Nil(t, err)
 
 		listener := &envoy_config_listener.Listener{}
@@ -87,11 +87,11 @@ func TestNewListener(t *testing.T) {
 		res1, err1 := newListener("dummy-name", "dummy-secret-namespace", true, map[model.TLSSecret][]string{
 			{Name: "dummy-secret-1", Namespace: "dummy-namespace"}: {"dummy.server.com"},
 			{Name: "dummy-secret-2", Namespace: "dummy-namespace"}: {"dummy.anotherserver.com"},
-		}, nil)
+		}, nil, true, true)
 		res2, err2 := newListener("dummy-name", "dummy-secret-namespace", true, map[model.TLSSecret][]string{
 			{Name: "dummy-secret-2", Namespace: "dummy-namespace"}: {"dummy.anotherserver.com"},
 			{Name: "dummy-secret-1", Namespace: "dummy-namespace"}: {"dummy.server.com"},
-		}, nil)
+		}, nil, true, true)
 
 		require.NoError(t, err1)
 		require.NoError(t, err2)
@@ -106,7 +106,7 @@ func TestNewListener(t *testing.T) {
 		res, err := newListener("dummy-name", "dummy-secret-namespace", true, map[model.TLSSecret][]string{
 			{Name: "dummy-secret-1", Namespace: "dummy-namespace"}: {"dummy.server.com"},
 			{Name: "dummy-secret-2", Namespace: "dummy-namespace"}: {"dummy.anotherserver.com"},
-		}, nil)
+		}, nil, true, true)
 		require.Nil(t, err)
 
 		listener := &envoy_config_listener.Listener{}
@@ -162,6 +162,7 @@ func TestNewListener(t *testing.T) {
 					"example.com",
 				},
 			},
+			true, true,
 		)
 		require.Nil(t, err)
 
@@ -190,6 +191,7 @@ func TestNewListener(t *testing.T) {
 					"foo.bar",
 				},
 			},
+			true, true,
 		)
 		res2, err2 := newListener("dummy-name",
 			"",
@@ -204,6 +206,7 @@ func TestNewListener(t *testing.T) {
 					"example.com",
 				},
 			},
+			true, true,
 		)
 		require.Nil(t, err1)
 		require.Nil(t, err2)
@@ -215,7 +218,7 @@ func TestNewListener(t *testing.T) {
 	})
 
 	t.Run("TLS passthrough with Proxy Protocol", func(t *testing.T) {
-		res, err := newListener("dummy-name", "", false, nil, map[string][]string{"dummy-namespace/dummy-service:443": {"example.org", "example.com"}}, WithProxyProtocol())
+		res, err := newListener("dummy-name", "", false, nil, map[string][]string{"dummy-namespace/dummy-service:443": {"example.org", "example.com"}}, true, true, WithProxyProtocol())
 		require.Nil(t, err)
 
 		listener := &envoy_config_listener.Listener{}
@@ -250,7 +253,7 @@ func TestNewListener(t *testing.T) {
 					"example.org",
 					"example.com",
 				},
-			},
+			}, true, true,
 		)
 		require.Nil(t, err)
 
@@ -276,7 +279,7 @@ func TestNewListener(t *testing.T) {
 	})
 
 	t.Run("without TLS with ALPN", func(t *testing.T) {
-		res, err := newListener("dummy-name", "dummy-secret-namespace", true, nil, nil, WithAlpn())
+		res, err := newListener("dummy-name", "dummy-secret-namespace", true, nil, nil, true, true, WithAlpn())
 		require.Nil(t, err)
 
 		listener := &envoy_config_listener.Listener{}
@@ -297,6 +300,7 @@ func TestNewListener(t *testing.T) {
 				{Name: "dummy-secret-1", Namespace: "dummy-namespace"}: {"dummy.server.com"},
 			},
 			nil,
+			true, true,
 			WithAlpn(),
 		)
 		require.Nil(t, err)

operator/pkg/model/translation/gateway-api/testdata/host_network/basic_http_listener/with_external_traffic_policy/cec-output.yaml

@@ -54,8 +54,6 @@ spec:
               prefixLen: 16
             - addressPrefix: 127.0.0.1
               prefixLen: 32
-            - addressPrefix: ::1
-              prefixLen: 128
           rds:
             routeConfigName: listener-insecure
           statPrefix: listener-insecure

operator/pkg/model/translation/gateway-api/testdata/host_network/basic_http_listener/without_external_traffic_policy/cec-output.yaml

@@ -54,8 +54,6 @@ spec:
               prefixLen: 16
             - addressPrefix: 127.0.0.1
               prefixLen: 32
-            - addressPrefix: ::1
-              prefixLen: 128
           rds:
             routeConfigName: listener-insecure
           statPrefix: listener-insecure

operator/pkg/model/translation/gateway-api/testdata/host_network/basic_http_listener_with_different_port/with_external_traffic_policy/cec-output.yaml

@@ -54,8 +54,6 @@ spec:
               prefixLen: 16
             - addressPrefix: 127.0.0.1
               prefixLen: 32
-            - addressPrefix: ::1
-              prefixLen: 128
           rds:
             routeConfigName: listener-insecure
           statPrefix: listener-insecure

operator/pkg/model/translation/gateway-api/testdata/host_network/basic_http_listener_with_different_port/without_external_traffic_policy/cec-output.yaml

@@ -54,8 +54,6 @@ spec:
               prefixLen: 16
             - addressPrefix: 127.0.0.1
               prefixLen: 32
-            - addressPrefix: ::1
-              prefixLen: 128
           rds:
             routeConfigName: listener-insecure
           statPrefix: listener-insecure