feat(kuma-cp): require kuma.io/origin: zone label when creating zon…

…e-origination policies (#8873)

Signed-off-by: Ilya Lobkov <[email protected]>

api/mesh/v1alpha1/dataplane_helpers.go

@@ -50,11 +50,25 @@ const (
 
 	// ResourceOriginLabel is a standard label that has information about the origin of the resource.
 	// It can be either "global" or "zone".
-	ResourceOriginLabel  = "kuma.io/origin"
-	ResourceOriginGlobal = "global"
-	ResourceOriginZone   = "zone"
+	ResourceOriginLabel = "kuma.io/origin"
 )
 
+type ResourceOrigin string
+
+const (
+	GlobalResourceOrigin ResourceOrigin = "global"
+	ZoneResourceOrigin   ResourceOrigin = "zone"
+)
+
+func (o ResourceOrigin) IsValid() error {
+	switch o {
+	case GlobalResourceOrigin, ZoneResourceOrigin:
+		return nil
+	default:
+		return errors.Errorf("unknown resource origin %q", o)
+	}
+}
+
 type ProxyType string
 
 const (

docs/generated/raw/kuma-cp.yaml

@@ -530,6 +530,8 @@ multizone:
       # Response backoff is a time Zone CP waits before sending ACK/NACK.
       # This is a way to slow down Global CP from sending resources too often.
       responseBackoff: 0s # ENV: KUMA_MULTIZONE_ZONE_KDS_RESPONSE_BACKOFF
+    # disableOriginLabelValidation disables validation of the origin label when applying resources on Zone CP
+    disableOriginLabelValidation: false # ENV: KUMA_MULTIZONE_ZONE_DISABLE_ORIGIN_LABEL_VALIDATION
 
 # Diagnostics configuration
 diagnostics:

pkg/api-server/resource_endpoints.go

@@ -51,15 +51,16 @@ const (
 )
 
 type resourceEndpoints struct {
-	mode               config_core.CpMode
-	federatedZone      bool
-	zoneName           string
-	resManager         manager.ResourceManager
-	descriptor         model.ResourceTypeDescriptor
-	resourceAccess     access.ResourceAccess
-	k8sMapper          k8s.ResourceMapperFunc
-	filter             func(request *restful.Request) (store.ListFilterFunc, error)
-	meshContextBuilder xds_context.MeshContextBuilder
+	mode                         config_core.CpMode
+	federatedZone                bool
+	zoneName                     string
+	resManager                   manager.ResourceManager
+	descriptor                   model.ResourceTypeDescriptor
+	resourceAccess               access.ResourceAccess
+	k8sMapper                    k8s.ResourceMapperFunc
+	filter                       func(request *restful.Request) (store.ListFilterFunc, error)
+	meshContextBuilder           xds_context.MeshContextBuilder
+	disableOriginLabelValidation bool
 }
 
 func typeToLegacyOverviewPath(resourceType model.ResourceType) string {
@@ -442,7 +443,7 @@ func (r *resourceEndpoints) validateResourceRequest(request *restful.Request, re
 		err.AddViolation("mesh", "mesh from the URL has to be the same as in body")
 	}
 
-	err.AddError("labels", r.validateLabels(resourceMeta.Labels))
+	err.AddError("labels", r.validateLabels(resourceMeta))
 
 	if create {
 		err.AddError("", mesh.ValidateMeta(resourceMeta, r.descriptor.Scope))
@@ -456,13 +457,27 @@ func (r *resourceEndpoints) validateResourceRequest(request *restful.Request, re
 	return err.OrNil()
 }
 
-func (r *resourceEndpoints) validateLabels(labels map[string]string) validators.ValidationError {
+func (r *resourceEndpoints) validateLabels(rm model.ResourceMeta) validators.ValidationError {
 	var err validators.ValidationError
-	for _, k := range maps.SortedKeys(labels) {
+
+	origin, ok := model.ResourceOrigin(rm)
+	if ok {
+		if oerr := origin.IsValid(); oerr != nil {
+			err.AddViolationAt(validators.Root().Key(mesh_proto.ResourceOriginLabel), oerr.Error())
+		}
+	}
+
+	if !r.disableOriginLabelValidation && r.federatedZone && r.descriptor.IsPluginOriginated {
+		if !ok || origin != mesh_proto.ZoneResourceOrigin {
+			err.AddViolationAt(validators.Root().Key(mesh_proto.ResourceOriginLabel), fmt.Sprintf("the origin label must be set to '%s'", mesh_proto.ZoneResourceOrigin))
+		}
+	}
+
+	for _, k := range maps.SortedKeys(rm.GetLabels()) {
 		for _, msg := range validation.IsQualifiedName(k) {
 			err.AddViolationAt(validators.Root().Key(k), msg)
 		}
-		for _, msg := range validation.IsValidLabelValue(labels[k]) {
+		for _, msg := range validation.IsValidLabelValue(rm.GetLabels()[k]) {
 			err.AddViolationAt(validators.Root().Key(k), msg)
 		}
 	}

pkg/api-server/server.go

@@ -263,14 +263,15 @@ func addResourcesEndpoints(
 			definition.ReadOnly = true
 		}
 		endpoints := resourceEndpoints{
-			k8sMapper:          k8sMapper,
-			mode:               cfg.Mode,
-			federatedZone:      cfg.IsFederatedZoneCP(),
-			resManager:         resManager,
-			descriptor:         definition,
-			resourceAccess:     resourceAccess,
-			filter:             filters.Resource(definition),
-			meshContextBuilder: meshContextBuilder,
+			k8sMapper:                    k8sMapper,
+			mode:                         cfg.Mode,
+			federatedZone:                cfg.IsFederatedZoneCP(),
+			resManager:                   resManager,
+			descriptor:                   definition,
+			resourceAccess:               resourceAccess,
+			filter:                       filters.Resource(definition),
+			meshContextBuilder:           meshContextBuilder,
+			disableOriginLabelValidation: cfg.Multizone.Zone.DisableOriginLabelValidation,
 		}
 		if cfg.Mode == config_core.Zone && cfg.Multizone != nil && cfg.Multizone.Zone != nil {
 			endpoints.zoneName = cfg.Multizone.Zone.Name

pkg/config/app/kuma-cp/kuma-cp.defaults.yaml

@@ -530,6 +530,8 @@ multizone:
       # Response backoff is a time Zone CP waits before sending ACK/NACK.
       # This is a way to slow down Global CP from sending resources too often.
       responseBackoff: 0s # ENV: KUMA_MULTIZONE_ZONE_KDS_RESPONSE_BACKOFF
+    # disableOriginLabelValidation disables validation of the origin label when applying resources on Zone CP
+    disableOriginLabelValidation: false # ENV: KUMA_MULTIZONE_ZONE_DISABLE_ORIGIN_LABEL_VALIDATION
 
 # Diagnostics configuration
 diagnostics:

pkg/config/loader_test.go

@@ -587,6 +587,7 @@ multizone:
       nackBackoff: 21s
       responseBackoff: 2s
       tlsSkipVerify: true
+    disableOriginLabelValidation: true
 dnsServer:
   domain: test-domain
   CIDR: 127.1.0.0/16
@@ -890,6 +891,7 @@ tracing:
 				"KUMA_MULTIZONE_ZONE_KDS_NACK_BACKOFF":                                                     "21s",
 				"KUMA_MULTIZONE_ZONE_KDS_RESPONSE_BACKOFF":                                                 "2s",
 				"KUMA_MULTIZONE_ZONE_KDS_TLS_SKIP_VERIFY":                                                  "true",
+				"KUMA_MULTIZONE_ZONE_DISABLE_ORIGIN_LABEL_VALIDATION":                                      "true",
 				"KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED":                                                      "true",
 				"KUMA_MULTIZONE_GLOBAL_KDS_ZONE_INSIGHT_FLUSH_INTERVAL":                                    "5s",
 				"KUMA_DEFAULTS_SKIP_MESH_CREATION":                                                         "true",

pkg/config/multizone/multicluster.go

@@ -61,6 +61,8 @@ type ZoneConfig struct {
 	GlobalAddress string `json:"globalAddress,omitempty" envconfig:"kuma_multizone_zone_global_address"`
 	// KDS Configuration
 	KDS *KdsClientConfig `json:"kds,omitempty"`
+	// DisableOriginLabelValidation disables validation of the origin label when applying resources on Zone CP
+	DisableOriginLabelValidation bool `json:"disableOriginLabelValidation,omitempty" envconfig:"kuma_multizone_zone_disable_origin_label_validation"`
 }
 
 func (r *ZoneConfig) Sanitize() {
@@ -118,6 +120,7 @@ func DefaultZoneConfig() *ZoneConfig {
 			MsgSendTimeout:  config_types.Duration{Duration: 60 * time.Second},
 			NackBackoff:     config_types.Duration{Duration: 5 * time.Second},
 		},
+		DisableOriginLabelValidation: false,
 	}
 }
 

pkg/core/resources/model/resource.go

@@ -399,11 +399,11 @@ func IsReferenced(refMeta ResourceMeta, refName string, resourceMeta ResourceMet
 func IsLocallyOriginated(mode config_core.CpMode, r Resource) bool {
 	switch mode {
 	case config_core.Global:
-		origin, ok := r.GetMeta().GetLabels()[mesh_proto.ResourceOriginLabel]
-		return !ok || origin == mesh_proto.ResourceOriginGlobal
+		origin, ok := ResourceOrigin(r.GetMeta())
+		return !ok || origin == mesh_proto.GlobalResourceOrigin
 	case config_core.Zone:
-		origin, ok := r.GetMeta().GetLabels()[mesh_proto.ResourceOriginLabel]
-		return !ok || origin == mesh_proto.ResourceOriginZone
+		origin, ok := ResourceOrigin(r.GetMeta())
+		return !ok || origin == mesh_proto.ZoneResourceOrigin
 	default:
 		return true
 	}
@@ -419,6 +419,13 @@ func GetDisplayName(r Resource) string {
 	return r.GetMeta().GetName()
 }
 
+func ResourceOrigin(rm ResourceMeta) (mesh_proto.ResourceOrigin, bool) {
+	if labels := rm.GetLabels(); labels != nil && labels[mesh_proto.ResourceOriginLabel] != "" {
+		return mesh_proto.ResourceOrigin(labels[mesh_proto.ResourceOriginLabel]), true
+	}
+	return "", false
+}
+
 // ZoneOfResource returns zone from which the resource was synced to Global CP
 // There is no information in the resource itself whether the resource is synced or created on the CP.
 // Therefore, it's a caller responsibility to make use it only on synced resources.

pkg/kds/context/context.go

@@ -61,7 +61,7 @@ func DefaultContext(
 	}
 
 	globalMappers := []reconcile.ResourceMapper{
-		UpdateResourceMeta(util.WithLabel(mesh_proto.ResourceOriginLabel, mesh_proto.ResourceOriginGlobal)),
+		UpdateResourceMeta(util.WithLabel(mesh_proto.ResourceOriginLabel, string(mesh_proto.GlobalResourceOrigin))),
 		reconcile.If(
 			reconcile.And(
 				reconcile.TypeIs(system.GlobalSecretType),
@@ -83,7 +83,7 @@ func DefaultContext(
 
 	zoneMappers := []reconcile.ResourceMapper{
 		UpdateResourceMeta(
-			util.WithLabel(mesh_proto.ResourceOriginLabel, mesh_proto.ResourceOriginZone),
+			util.WithLabel(mesh_proto.ResourceOriginLabel, string(mesh_proto.ZoneResourceOrigin)),
 			util.WithLabel(mesh_proto.ZoneTag, cfg.Multizone.Zone.Name),
 		),
 		MapInsightResourcesZeroGeneration,

pkg/kds/global/components.go

@@ -236,7 +236,7 @@ func Callbacks(s sync_store.ResourceSyncer, k8sStore bool, kubeFactory resources
 			for _, r := range rs.GetItems() {
 				r.SetMeta(util.CloneResourceMeta(r.GetMeta(),
 					util.WithLabel(mesh_proto.ZoneTag, clusterName),
-					util.WithLabel(mesh_proto.ResourceOriginLabel, mesh_proto.ResourceOriginZone),
+					util.WithLabel(mesh_proto.ResourceOriginLabel, string(mesh_proto.ZoneResourceOrigin)),
 				))
 			}
 

pkg/kds/v2/client/zone_sync_test.go

@@ -135,7 +135,7 @@ var _ = Describe("Zone Delta Sync", func() {
 
 		Expect(actual.Items[0].Spec).To(Equal(samples.Mesh1))
 		Expect(actual.Items[0].Meta.GetLabels()).To(Equal(map[string]string{
-			mesh_proto.ResourceOriginLabel: mesh_proto.ResourceOriginGlobal,
+			mesh_proto.ResourceOriginLabel: string(mesh_proto.GlobalResourceOrigin),
 			"foo":                          "bar",
 		}))
 	})
@@ -162,7 +162,7 @@ var _ = Describe("Zone Delta Sync", func() {
 
 		Expect(actual.Items[0].Spec).To(Equal(samples.Mesh1))
 		Expect(actual.Items[0].Meta.GetLabels()).To(Equal(map[string]string{
-			mesh_proto.ResourceOriginLabel: mesh_proto.ResourceOriginGlobal,
+			mesh_proto.ResourceOriginLabel: string(mesh_proto.GlobalResourceOrigin),
 			"foo":                          "bar",
 		}))
 
@@ -184,7 +184,7 @@ var _ = Describe("Zone Delta Sync", func() {
 			// then zone store should have updated mesh
 			g.Expect(actual.Items[0].GetSpec().(*mesh_proto.Mesh).Mtls).To(BeNil())
 			g.Expect(actual.Items[0].GetMeta().GetLabels()).To(Equal(map[string]string{
-				mesh_proto.ResourceOriginLabel: mesh_proto.ResourceOriginGlobal,
+				mesh_proto.ResourceOriginLabel: string(mesh_proto.GlobalResourceOrigin),
 				"foo":                          "barbar",
 				"newlabel":                     "newvalue",
 			}))
@@ -299,7 +299,7 @@ var _ = Describe("Zone Delta Sync", func() {
 			g.Expect(err).ToNot(HaveOccurred())
 			g.Expect(actual.Items).To(HaveLen(1))
 			g.Expect(actual.Items[0].GetMeta().GetLabels()).To(Equal(map[string]string{
-				mesh_proto.ResourceOriginLabel: mesh_proto.ResourceOriginGlobal,
+				mesh_proto.ResourceOriginLabel: string(mesh_proto.GlobalResourceOrigin),
 			}))
 		}, "5s", "100ms").Should(Succeed())
 	})

pkg/kds/v2/store/sync.go

@@ -345,7 +345,7 @@ func GlobalSyncCallback(
 			for _, r := range upstream.AddedResources.GetItems() {
 				r.SetMeta(util.CloneResourceMeta(r.GetMeta(),
 					util.WithLabel(mesh_proto.ZoneTag, upstream.ControlPlaneId),
-					util.WithLabel(mesh_proto.ResourceOriginLabel, mesh_proto.ResourceOriginZone),
+					util.WithLabel(mesh_proto.ResourceOriginLabel, string(mesh_proto.ZoneResourceOrigin)),
 				))
 			}
 

pkg/plugins/policies/core/matchers/dataplane.go

@@ -261,10 +261,11 @@ func (b ByTargetRef) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
 // If we assign numbers to origins like Global=-1, Zone=1, Unknown=0, then we can compare them as numbers
 // and get the same result as in the table above.
 func originToNumber(r core_model.Resource) int {
-	switch r.GetMeta().GetLabels()[mesh_proto.ResourceOriginLabel] {
-	case mesh_proto.ResourceOriginGlobal:
+	origin, _ := core_model.ResourceOrigin(r.GetMeta())
+	switch origin {
+	case mesh_proto.GlobalResourceOrigin:
 		return -1
-	case mesh_proto.ResourceOriginZone:
+	case mesh_proto.ZoneResourceOrigin:
 		return 1
 	default:
 		return 0

pkg/plugins/runtime/k8s/plugin.go

@@ -243,7 +243,7 @@ func addValidators(mgr kube_ctrl.Manager, rt core_runtime.Runtime, converter k8s
 	}
 
 	allowedUsers := append(rt.Config().Runtime.Kubernetes.AllowedUsers, rt.Config().Runtime.Kubernetes.ServiceAccountName, "system:serviceaccount:kube-system:generic-garbage-collector")
-	handler := k8s_webhooks.NewValidatingWebhook(converter, core_registry.Global(), k8s_registry.Global(), rt.Config().Mode, rt.Config().IsFederatedZoneCP(), allowedUsers)
+	handler := k8s_webhooks.NewValidatingWebhook(converter, core_registry.Global(), k8s_registry.Global(), rt.Config().Mode, rt.Config().IsFederatedZoneCP(), allowedUsers, rt.Config().Multizone.Zone.DisableOriginLabelValidation)
 	composite.AddValidator(handler)
 
 	k8sMeshValidator := k8s_webhooks.NewMeshValidatorWebhook(rt.ResourceValidators().Mesh, converter, rt.Config().Store.UnsafeDelete)