Skip to content

Add feature flag to secure cluster connections #621

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
koct9i wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add feature flag to secure cluster connections #621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions api/v1/ytsaurus_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -808,6 +808,8 @@ type ClusterFeatures struct {
HTTPProxyHaveChytAddress bool `json:"httpProxyHaveChytAddress,omitempty"`
// HTTP proxies have "https" address. Use HTTPS for all communications.
HTTPProxyHaveHTTPSAddress bool `json:"httpProxyHaveHttpsAddress,omitempty"`
// Validate that only secure transports are allowed for cluster connections.
SecureClusterTransports bool `json:"secureClusterTransports,omitempty"`
}

// CommonSpec is a set of fields shared between `YtsaurusSpec` and `Remote*NodesSpec`.
Expand Down
116 changes: 102 additions & 14 deletions api/v1/ytsaurus_webhook.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,18 +68,61 @@ func (r *Ytsaurus) SetupWebhookWithManager(mgr ctrl.Manager) error {

//////////////////////////////////////////////////

func (r *baseValidator) validateTransportSecurity(spec *RPCTransportSpec, commonSpec *CommonSpec, path *field.Path) field.ErrorList {
var allErrors field.ErrorList

features := ptr.Deref(commonSpec.ClusterFeatures, ClusterFeatures{})

if spec == nil {
spec = commonSpec.NativeTransport
}

if spec == nil {
if features.SecureClusterTransports {
allErrors = append(allErrors, field.Required(path, "Secure cluster transport demands TLS setup"))
}
return allErrors
}

secretPath := path.Child("tlsSecret")
clientSecretPath := path.Child("tlsClientSecret")
if spec.TLSSecret == nil && (spec.TLSRequired || !spec.TLSInsecure) {
allErrors = append(allErrors, field.Required(secretPath, "TLS certificate for native transport is required"))
}
if spec.TLSClientSecret == nil && (spec.TLSRequired && !spec.TLSInsecure) {
allErrors = append(allErrors, field.Required(clientSecretPath, "Client TLS certificate for native transport is required"))
}

if features.SecureClusterTransports {
if !spec.TLSRequired {
allErrors = append(allErrors, field.Forbidden(path.Child("tlsRequired"), "Secure cluster transport demands TLS-only native transport"))
}
if spec.TLSInsecure {
allErrors = append(allErrors, field.Forbidden(path.Child("tlsInsecure"), "Secure cluster transport demands TLS certificate validation"))
}
if spec.TLSSecret == nil {
allErrors = append(allErrors, field.Required(secretPath, "Secure cluster transport demands TLS certificate for native transport"))
}
if spec.TLSClientSecret == nil {
allErrors = append(allErrors, field.Required(clientSecretPath, "Secure cluster transport demands client TLS certificate for native transport"))
}
}

return allErrors
}

func (r *ytsaurusValidator) validateDiscovery(newYtsaurus *Ytsaurus) field.ErrorList {
var allErrors field.ErrorList

allErrors = append(allErrors, r.validateInstanceSpec(newYtsaurus.Spec.Discovery.InstanceSpec, field.NewPath("spec").Child("discovery"))...)
allErrors = append(allErrors, r.validateInstanceSpec(newYtsaurus.Spec.Discovery.InstanceSpec, &newYtsaurus.Spec.CommonSpec, field.NewPath("spec").Child("discovery"))...)

return allErrors
}

func (r *ytsaurusValidator) validateMasterSpec(newYtsaurus *Ytsaurus, mastersSpec, oldMastersSpec *MastersSpec, path *field.Path) field.ErrorList {
var allErrors field.ErrorList

allErrors = append(allErrors, r.validateInstanceSpec(mastersSpec.InstanceSpec, path)...)
allErrors = append(allErrors, r.validateInstanceSpec(mastersSpec.InstanceSpec, &newYtsaurus.Spec.CommonSpec, path)...)
allErrors = append(allErrors, r.validateHostAddresses(newYtsaurus, mastersSpec, path)...)

if FindFirstLocation(mastersSpec.Locations, LocationTypeMasterChangelogs) == nil {
Expand Down Expand Up @@ -177,14 +220,21 @@ func (r *ytsaurusValidator) validateHTTPProxies(newYtsaurus *Ytsaurus) field.Err
}
httpRoles[hp.Role] = true

allErrors = append(allErrors, r.validateInstanceSpec(hp.InstanceSpec, path)...)
allErrors = append(allErrors, r.validateInstanceSpec(hp.InstanceSpec, &newYtsaurus.Spec.CommonSpec, path)...)

if features.HTTPProxyHaveHTTPSAddress && hp.Transport.HTTPSSecret == nil {
allErrors = append(allErrors, field.Required(
path.Child("transport").Child("httpsSecret"),
"Cluster feature httpProxyHaveHttpsAddress requires HTTPS for all HTTP proxies",
))
}

if features.SecureClusterTransports && !hp.Transport.DisableHTTP {
allErrors = append(allErrors, field.Forbidden(
path.Child("transport").Child("disableHttp"),
"Secure cluster transport demands HTTPS-only proxies",
))
}
}

if !hasDefaultHTTPProxy {
Expand All @@ -199,6 +249,7 @@ func (r *ytsaurusValidator) validateHTTPProxies(newYtsaurus *Ytsaurus) field.Err
func (r *ytsaurusValidator) validateRPCProxies(newYtsaurus *Ytsaurus) field.ErrorList {
var allErrors field.ErrorList

features := ptr.Deref(newYtsaurus.Spec.ClusterFeatures, ClusterFeatures{})
rpcRoles := make(map[string]bool)
for i, rp := range newYtsaurus.Spec.RPCProxies {
path := field.NewPath("spec").Child("rpcProxies").Index(i)
Expand All @@ -207,7 +258,24 @@ func (r *ytsaurusValidator) validateRPCProxies(newYtsaurus *Ytsaurus) field.Erro
}
rpcRoles[rp.Role] = true

allErrors = append(allErrors, r.validateInstanceSpec(rp.InstanceSpec, path)...)
allErrors = append(allErrors, r.validateInstanceSpec(rp.InstanceSpec, &newYtsaurus.Spec.CommonSpec, path)...)

transportPath := path.Child("transport")
if rp.Transport.TLSRequired && rp.Transport.TLSSecret == nil {
allErrors = append(allErrors, field.Required(transportPath.Child("tlsSecret"), "TLS-only RPC proxy requires certificate"))
}

if features.SecureClusterTransports {
if !rp.Transport.TLSRequired {
allErrors = append(allErrors, field.Required(transportPath.Child("tlsRequired"), "Secure cluster transport demands TLS-only RPC proxies"))
}
if rp.Transport.TLSInsecure {
allErrors = append(allErrors, field.Forbidden(transportPath.Child("tlsInsecure"), "Secure cluster transport demands TLS certificate validation"))
}
if rp.Transport.TLSSecret == nil {
allErrors = append(allErrors, field.Required(transportPath.Child("tlsSecret"), "Secure cluster transport demands RPC proxy certificate"))
}
}
}

return allErrors
Expand All @@ -224,7 +292,7 @@ func (r *ytsaurusValidator) validateTCPProxies(newYtsaurus *Ytsaurus) field.Erro
}
tcpRoles[rp.Role] = true

allErrors = append(allErrors, r.validateInstanceSpec(rp.InstanceSpec, path)...)
allErrors = append(allErrors, r.validateInstanceSpec(rp.InstanceSpec, &newYtsaurus.Spec.CommonSpec, path)...)
}

return allErrors
Expand All @@ -242,7 +310,7 @@ func (r *ytsaurusValidator) validateDataNodes(newYtsaurus *Ytsaurus) field.Error
}
names[dn.Name] = true

allErrors = append(allErrors, r.validateInstanceSpec(dn.InstanceSpec, path)...)
allErrors = append(allErrors, r.validateInstanceSpec(dn.InstanceSpec, &newYtsaurus.Spec.CommonSpec, path)...)

if FindFirstLocation(dn.Locations, LocationTypeChunkStore) == nil {
allErrors = append(allErrors, field.NotFound(path.Child("locations"), LocationTypeChunkStore))
Expand Down Expand Up @@ -311,7 +379,7 @@ func (r *ytsaurusValidator) validateExecNodes(newYtsaurus *Ytsaurus) field.Error
}
names[en.Name] = true

allErrors = append(allErrors, r.validateInstanceSpec(en.InstanceSpec, path)...)
allErrors = append(allErrors, r.validateInstanceSpec(en.InstanceSpec, &newYtsaurus.Spec.CommonSpec, path)...)

if FindFirstLocation(en.Locations, LocationTypeChunkCache) == nil {
allErrors = append(allErrors, field.NotFound(path.Child("locations"), LocationTypeChunkCache))
Expand Down Expand Up @@ -344,7 +412,7 @@ func (r *ytsaurusValidator) validateSchedulers(newYtsaurus *Ytsaurus) field.Erro

if newYtsaurus.Spec.Schedulers != nil {
path := field.NewPath("spec").Child("schedulers")
allErrors = append(allErrors, r.validateInstanceSpec(newYtsaurus.Spec.Schedulers.InstanceSpec, path)...)
allErrors = append(allErrors, r.validateInstanceSpec(newYtsaurus.Spec.Schedulers.InstanceSpec, &newYtsaurus.Spec.CommonSpec, path)...)

if newYtsaurus.Spec.ControllerAgents == nil {
allErrors = append(allErrors, field.Required(field.NewPath("spec").Child("controllerAgents"),
Expand All @@ -360,7 +428,7 @@ func (r *ytsaurusValidator) validateControllerAgents(newYtsaurus *Ytsaurus) fiel

if newYtsaurus.Spec.ControllerAgents != nil {
path := field.NewPath("spec").Child("controllerAgents")
allErrors = append(allErrors, r.validateInstanceSpec(newYtsaurus.Spec.ControllerAgents.InstanceSpec, path)...)
allErrors = append(allErrors, r.validateInstanceSpec(newYtsaurus.Spec.ControllerAgents.InstanceSpec, &newYtsaurus.Spec.CommonSpec, path)...)

if newYtsaurus.Spec.Schedulers == nil {
allErrors = append(allErrors, field.Required(field.NewPath("spec").Child("schedulers"),
Expand All @@ -383,7 +451,7 @@ func (r *ytsaurusValidator) validateTabletNodes(newYtsaurus *Ytsaurus) field.Err
}
names[tn.Name] = true

allErrors = append(allErrors, r.validateInstanceSpec(tn.InstanceSpec, path)...)
allErrors = append(allErrors, r.validateInstanceSpec(tn.InstanceSpec, &newYtsaurus.Spec.CommonSpec, path)...)
}

return allErrors
Expand Down Expand Up @@ -413,7 +481,7 @@ func (r *ytsaurusValidator) validateQueryTrackers(newYtsaurus *Ytsaurus) field.E

if newYtsaurus.Spec.QueryTrackers != nil {
path := field.NewPath("spec").Child("queryTrackers")
allErrors = append(allErrors, r.validateInstanceSpec(newYtsaurus.Spec.QueryTrackers.InstanceSpec, path)...)
allErrors = append(allErrors, r.validateInstanceSpec(newYtsaurus.Spec.QueryTrackers.InstanceSpec, &newYtsaurus.Spec.CommonSpec, path)...)

if len(newYtsaurus.Spec.TabletNodes) == 0 {
allErrors = append(allErrors, field.Required(field.NewPath("spec").Child("tabletNodes"),
Expand All @@ -434,7 +502,7 @@ func (r *ytsaurusValidator) validateQueueAgents(newYtsaurus *Ytsaurus) field.Err

if newYtsaurus.Spec.QueueAgents != nil {
path := field.NewPath("spec").Child("queueAgents")
allErrors = append(allErrors, r.validateInstanceSpec(newYtsaurus.Spec.QueueAgents.InstanceSpec, path)...)
allErrors = append(allErrors, r.validateInstanceSpec(newYtsaurus.Spec.QueueAgents.InstanceSpec, &newYtsaurus.Spec.CommonSpec, path)...)

if len(newYtsaurus.Spec.TabletNodes) == 0 {
allErrors = append(allErrors, field.Required(field.NewPath("spec").Child("tabletNodes"),
Expand Down Expand Up @@ -462,7 +530,7 @@ func (r *ytsaurusValidator) validateYQLAgents(newYtsaurus *Ytsaurus) field.Error

if newYtsaurus.Spec.YQLAgents != nil {
path := field.NewPath("spec").Child("YQLAgents")
allErrors = append(allErrors, r.validateInstanceSpec(newYtsaurus.Spec.YQLAgents.InstanceSpec, path)...)
allErrors = append(allErrors, r.validateInstanceSpec(newYtsaurus.Spec.YQLAgents.InstanceSpec, &newYtsaurus.Spec.CommonSpec, path)...)

if newYtsaurus.Spec.QueryTrackers == nil {
allErrors = append(allErrors, field.Required(field.NewPath("spec").Child("queryTrackers"),
Expand Down Expand Up @@ -493,7 +561,7 @@ func (r *ytsaurusValidator) validateUi(newYtsaurus *Ytsaurus) field.ErrorList {
return allErrors
}

func (r *baseValidator) validateInstanceSpec(instanceSpec InstanceSpec, path *field.Path) field.ErrorList {
func (r *baseValidator) validateInstanceSpec(instanceSpec InstanceSpec, commonSpec *CommonSpec, path *field.Path) field.ErrorList {
var allErrors field.ErrorList

allErrors = append(allErrors, v1validation.ValidateLabels(instanceSpec.PodLabels, path.Child("podLabels"))...)
Expand All @@ -504,6 +572,8 @@ func (r *baseValidator) validateInstanceSpec(instanceSpec InstanceSpec, path *fi
"EnableAntiAffinity is deprecated, use Affinity instead"))
}

allErrors = append(allErrors, r.validateTransportSecurity(instanceSpec.NativeTransport, commonSpec, path.Child("nativeTransport"))...)

if instanceSpec.Locations != nil {
for locationIdx, location := range instanceSpec.Locations {
inVolumeMount := false
Expand Down Expand Up @@ -554,7 +624,25 @@ func (r *baseValidator) validateCommonSpec(spec *CommonSpec) field.ErrorList {
var allErrors field.ErrorList
path := field.NewPath("spec")

if features := spec.ClusterFeatures; features != nil {
if features.SecureClusterTransports {
if !features.RPCProxyHavePublicAddress {
allErrors = append(allErrors, field.Required(
path.Child("clusterFeatures").Child("rpcProxyHavePublicAddress"),
"Secure cluster transport demands public address for RPC proxies",
))
}
if !features.HTTPProxyHaveHTTPSAddress {
allErrors = append(allErrors, field.Required(
path.Child("clusterFeatures").Child("httpProxyHaveHttpsAddress"),
"Secure cluster transport demands HTTPS for HTTP proxies",
))
}
}
}

allErrors = append(allErrors, validation.ValidateAnnotations(spec.ExtraPodAnnotations, path.Child("extraPodAnnotations"))...)
allErrors = append(allErrors, r.validateTransportSecurity(spec.NativeTransport, spec, path.Child("nativeTransport"))...)

return allErrors
}
Expand Down
4 changes: 4 additions & 0 deletions config/crd/bases/cluster.ytsaurus.tech_offshoredatagateways.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -771,6 +771,10 @@ spec:
rpcProxyHavePublicAddress:
description: RPC proxies have "public_rpc" address.
type: boolean
secureClusterTransports:
description: Validate that only secure transports are allowed
for cluster connections.
type: boolean
type: object
configOverrides:
description: LocalObjectReference contains enough information to let
Expand Down
4 changes: 4 additions & 0 deletions config/crd/bases/cluster.ytsaurus.tech_remotedatanodes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -769,6 +769,10 @@ spec:
rpcProxyHavePublicAddress:
description: RPC proxies have "public_rpc" address.
type: boolean
secureClusterTransports:
description: Validate that only secure transports are allowed
for cluster connections.
type: boolean
type: object
configOverrides:
description: LocalObjectReference contains enough information to let
Expand Down
4 changes: 4 additions & 0 deletions config/crd/bases/cluster.ytsaurus.tech_remoteexecnodes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -769,6 +769,10 @@ spec:
rpcProxyHavePublicAddress:
description: RPC proxies have "public_rpc" address.
type: boolean
secureClusterTransports:
description: Validate that only secure transports are allowed
for cluster connections.
type: boolean
type: object
configOverrides:
description: LocalObjectReference contains enough information to let
Expand Down
4 changes: 4 additions & 0 deletions config/crd/bases/cluster.ytsaurus.tech_remotetabletnodes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -769,6 +769,10 @@ spec:
rpcProxyHavePublicAddress:
description: RPC proxies have "public_rpc" address.
type: boolean
secureClusterTransports:
description: Validate that only secure transports are allowed
for cluster connections.
type: boolean
type: object
configOverrides:
description: LocalObjectReference contains enough information to let
Expand Down
4 changes: 4 additions & 0 deletions config/crd/bases/cluster.ytsaurus.tech_ytsaurus.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1998,6 +1998,10 @@ spec:
rpcProxyHavePublicAddress:
description: RPC proxies have "public_rpc" address.
type: boolean
secureClusterTransports:
description: Validate that only secure transports are allowed
for cluster connections.
type: boolean
type: object
configOverrides:
description: LocalObjectReference contains enough information to let
Expand Down
1 change: 1 addition & 0 deletions docs/api.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,7 @@ _Appears in:_
| `rpcProxyHavePublicAddress` _boolean_ | RPC proxies have "public_rpc" address. Required for separated internal/public TLS CA. | | |
| `httpProxyHaveChytAddress` _boolean_ | HTTP proxies have "chyt_http_server" and "chyt_https_server". Opens ports for access to chyt via HTTP proxy. | | |
| `httpProxyHaveHttpsAddress` _boolean_ | HTTP proxies have "https" address. Use HTTPS for all communications. | | |
| `secureClusterTransports` _boolean_ | Validate that only secure transports are allowed for cluster connections. | | |


#### ClusterNodesSpec
Expand Down
1 change: 1 addition & 0 deletions pkg/testutil/spec_builders.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -361,6 +361,7 @@ func (b *YtsaurusBuilder) WithAllClusterFeatures() {
RPCProxyHavePublicAddress: true,
HTTPProxyHaveChytAddress: true,
HTTPProxyHaveHTTPSAddress: true,
SecureClusterTransports: false, // Turned off to increase coverage.
}
}

Expand Down
Loading
Loading