diff --git a/docs/user-guide/nginx-configuration/annotations-risk.md b/docs/user-guide/nginx-configuration/annotations-risk.md index aff9357b88..926137745a 100755 --- a/docs/user-guide/nginx-configuration/annotations-risk.md +++ b/docs/user-guide/nginx-configuration/annotations-risk.md @@ -87,7 +87,9 @@ | Proxy | proxy-redirect-to | Medium | location | | Proxy | proxy-request-buffering | Low | location | | Proxy | proxy-send-timeout | Low | location | +| ProxySSL | proxy-ssl-ca-configmap | Medium | ingress | | ProxySSL | proxy-ssl-ciphers | Medium | ingress | +| ProxySSL | proxy-ssl-client-secret | Medium | ingress | | ProxySSL | proxy-ssl-name | High | ingress | | ProxySSL | proxy-ssl-protocols | Low | ingress | | ProxySSL | proxy-ssl-secret | Medium | ingress | diff --git a/internal/ingress/annotations/proxyssl/main.go b/internal/ingress/annotations/proxyssl/main.go index c40bc85e03..aeb7918091 100644 --- a/internal/ingress/annotations/proxyssl/main.go +++ b/internal/ingress/annotations/proxyssl/main.go @@ -45,13 +45,15 @@ var ( ) const ( - proxySSLSecretAnnotation = "proxy-ssl-secret" - proxySSLCiphersAnnotation = "proxy-ssl-ciphers" - proxySSLProtocolsAnnotation = "proxy-ssl-protocols" - proxySSLNameAnnotation = "proxy-ssl-name" - proxySSLVerifyAnnotation = "proxy-ssl-verify" - proxySSLVerifyDepthAnnotation = "proxy-ssl-verify-depth" - proxySSLServerNameAnnotation = "proxy-ssl-server-name" + proxySSLSecretAnnotation = "proxy-ssl-secret" // DEPRECATED Use proxy-ssl-client-secret and proxy-ssl-ca-configmap instead + proxySSLClientSecretAnnotation = "proxy-ssl-client-secret" // #nosec + proxySSLCAConfigMapAnnotation = "proxy-ssl-ca-configmap" + proxySSLCiphersAnnotation = "proxy-ssl-ciphers" + proxySSLProtocolsAnnotation = "proxy-ssl-protocols" + proxySSLNameAnnotation = "proxy-ssl-name" + proxySSLVerifyAnnotation = "proxy-ssl-verify" + proxySSLVerifyDepthAnnotation = "proxy-ssl-verify-depth" + proxySSLServerNameAnnotation = "proxy-ssl-server-name" ) var proxySSLAnnotation = parser.Annotation{ @@ -61,11 +63,30 @@ var proxySSLAnnotation = parser.Annotation{ Validator: parser.ValidateRegex(parser.BasicCharsRegex, true), Scope: parser.AnnotationScopeIngress, Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation specifies a Secret with the certificate tls.crt, key tls.key in PEM format used for authentication to a proxied HTTPS server. + Documentation: `(DEPRECATED: Use proxy-ssl-client-secret and proxy-ssl-ca-configmap instead) + This annotation specifies a Secret with the certificate tls.crt, key tls.key in PEM format used for authentication to a proxied HTTPS server. It should also contain trusted CA certificates ca.crt in PEM format used to verify the certificate of the proxied HTTPS server. This annotation expects the Secret name in the form "namespace/secretName" Just secrets on the same namespace of the ingress can be used.`, }, + proxySSLClientSecretAnnotation: { + Validator: parser.ValidateRegex(parser.BasicCharsRegex, true), + Scope: parser.AnnotationScopeIngress, + Risk: parser.AnnotationRiskMedium, + Documentation: `This annotation specifies a Secret with the certificate tls.crt, key tls.key in PEM format used for authentication to a proxied HTTPS server. + If the annotation proxy-ssl-secret is also present, the tls.crt and tls.key from this secret will take precedence. + This annotation expects the Secret name in the form "namespace/secretName" + Just secrets on the same namespace of the ingress can be used.`, + }, + proxySSLCAConfigMapAnnotation: { + Validator: parser.ValidateRegex(parser.BasicCharsRegex, true), + Scope: parser.AnnotationScopeIngress, + Risk: parser.AnnotationRiskMedium, + Documentation: `This annotation specifies a ConfigMap with the trusted CA certificates ca.crt in PEM format used to verify the certificate of the proxied HTTPS server. + If the annotation proxy-ssl-secret is also present, ca tls.crt and ca.clr (revocation list) from this configMap will take precedence. + This annotation expects the ConfigMap name in the form "namespace/configMapName" + Just configMaps on the same namespace of the ingress can be used.`, + }, proxySSLCiphersAnnotation: { Validator: parser.ValidateRegex(proxySSLCiphersRegex, true), Scope: parser.AnnotationScopeIngress, @@ -107,16 +128,18 @@ var proxySSLAnnotation = parser.Annotation{ }, } -// Config contains the AuthSSLCert used for mutual authentication +// Config contains the Proxy SSL certificates and CAs used for mutual authentication // and the configured VerifyDepth type Config struct { resolver.AuthSSLCert - Ciphers string `json:"ciphers"` - Protocols string `json:"protocols"` - ProxySSLName string `json:"proxySSLName"` - Verify string `json:"verify"` - VerifyDepth int `json:"verifyDepth"` - ProxySSLServerName string `json:"proxySSLServerName"` + ProxySSLClientCert resolver.SSLClientCert `json:"proxySSLClientCert"` + ProxySSLCA resolver.SSLCA `json:"proxySSLCA"` + Ciphers string `json:"ciphers"` + Protocols string `json:"protocols"` + ProxySSLName string `json:"proxySSLName"` + Verify string `json:"verify"` + VerifyDepth int `json:"verifyDepth"` + ProxySSLServerName string `json:"proxySSLServerName"` } // Equal tests for equality between two Config types @@ -130,6 +153,12 @@ func (pssl1 *Config) Equal(pssl2 *Config) bool { if !(&pssl1.AuthSSLCert).Equal(&pssl2.AuthSSLCert) { return false } + if !(&pssl1.ProxySSLClientCert).Equal(&pssl2.ProxySSLClientCert) { + return false + } + if !(&pssl1.ProxySSLCA).Equal(&pssl2.ProxySSLCA) { + return false + } if pssl1.Ciphers != pssl2.Ciphers { return false } @@ -212,6 +241,50 @@ func (p proxySSL) Parse(ing *networking.Ingress) (interface{}, error) { } config.AuthSSLCert = *proxyCert + proxysslclientsecret, err := parser.GetStringAnnotation(proxySSLClientSecretAnnotation, ing, p.annotationConfig.Annotations) + if err != nil { + return &Config{}, err + } + + ns, _, err = k8s.ParseNameNS(proxysslclientsecret) + if err != nil { + return &Config{}, ing_errors.NewLocationDenied(err.Error()) + } + + // We don't accept different namespaces for secrets. + if !secCfg.AllowCrossNamespaceResources && ns != ing.Namespace { + return &Config{}, ing_errors.NewLocationDenied("cross namespace secrets are not supported") + } + + sslClientCert, err := p.r.GetSSLClientCert(proxysslclientsecret) + if err != nil { + e := fmt.Errorf("error obtaining ssl client certificate: %w", err) + return &Config{}, ing_errors.LocationDeniedError{Reason: e} + } + config.ProxySSLClientCert = *sslClientCert + + proxysslcaconfigmap, err := parser.GetStringAnnotation(proxySSLCAConfigMapAnnotation, ing, p.annotationConfig.Annotations) + if err != nil { + return &Config{}, err + } + + ns, _, err = k8s.ParseNameNS(proxysslcaconfigmap) + if err != nil { + return &Config{}, ing_errors.NewLocationDenied(err.Error()) + } + + // We don't accept different namespaces for configmaps. + if !secCfg.AllowCrossNamespaceResources && ns != ing.Namespace { + return &Config{}, ing_errors.NewLocationDenied("cross namespace configmaps are not supported") + } + + sslCA, err := p.r.GetSSLCA(proxysslcaconfigmap) + if err != nil { + e := fmt.Errorf("error obtaining ssl certificate authority: %w", err) + return &Config{}, ing_errors.LocationDeniedError{Reason: e} + } + config.ProxySSLCA = *sslCA + config.Ciphers, err = parser.GetStringAnnotation(proxySSLCiphersAnnotation, ing, p.annotationConfig.Annotations) if err != nil { if ing_errors.IsValidationError(err) { diff --git a/internal/ingress/annotations/proxyssl/main_test.go b/internal/ingress/annotations/proxyssl/main_test.go index cfa31f1d1f..ed834de220 100644 --- a/internal/ingress/annotations/proxyssl/main_test.go +++ b/internal/ingress/annotations/proxyssl/main_test.go @@ -28,11 +28,12 @@ import ( ) const ( - defaultDemoSecret = "default/demo-secret" - proxySslCiphers = "HIGH:-SHA" - off = "off" - sslServerName = "w00t" - defaultProtocol = "TLSv1.2 TLSv1.3" + defaultDemoSecret = "default/demo-secret" + defaultDemoConfigMap = "default/demo-configmap" + proxySslCiphers = "HIGH:-SHA" + off = "off" + sslServerName = "w00t" + defaultProtocol = "TLSv1.2 TLSv1.3" ) func buildIngress() *networking.Ingress { @@ -96,11 +97,37 @@ func (m mockSecret) GetAuthCertificate(name string) (*resolver.AuthSSLCert, erro }, nil } +// GetSSLClientCert resolves a given secret name into an SSL certificate. +func (m mockSecret) GetSSLClientCert(name string) (*resolver.SSLClientCert, error) { + if name != defaultDemoSecret { + return nil, errors.Errorf("there is no secret with name %v", name) + } + + return &resolver.SSLClientCert{ + Secret: defaultDemoSecret, + }, nil +} + +// GetSSLCA resolves a given configMap name into an SSL CA. +func (m mockSecret) GetSSLCA(name string) (*resolver.SSLCA, error) { + if name != defaultDemoConfigMap { + return nil, errors.Errorf("there is no configmap with name %v", name) + } + + return &resolver.SSLCA{ + ConfigMap: defaultDemoConfigMap, + CAFileName: "/ssl/ca.crt", + CASHA: "abc", + }, nil +} + func TestAnnotations(t *testing.T) { ing := buildIngress() data := map[string]string{} data[parser.GetAnnotationWithPrefix(proxySSLSecretAnnotation)] = defaultDemoSecret + data[parser.GetAnnotationWithPrefix(proxySSLClientSecretAnnotation)] = defaultDemoSecret + data[parser.GetAnnotationWithPrefix(proxySSLCAConfigMapAnnotation)] = defaultDemoConfigMap data[parser.GetAnnotationWithPrefix("proxy-ssl-ciphers")] = proxySslCiphers data[parser.GetAnnotationWithPrefix("proxy-ssl-name")] = "$host" data[parser.GetAnnotationWithPrefix("proxy-ssl-protocols")] = "TLSv1.3 TLSv1.2" @@ -126,10 +153,24 @@ func TestAnnotations(t *testing.T) { if err != nil { t.Errorf("unexpected error getting secret %v", err) } + clientSecret, err := fakeSecret.GetSSLClientCert(defaultDemoSecret) + if err != nil { + t.Errorf("unexpected error getting secret %v", err) + } + configMap, err := fakeSecret.GetSSLCA(defaultDemoConfigMap) + if err != nil { + t.Errorf("unexpected error getting configmap %v", err) + } if u.AuthSSLCert.Secret != secret.Secret { t.Errorf("expected %v but got %v", secret.Secret, u.AuthSSLCert.Secret) } + if u.ProxySSLClientCert.Secret != clientSecret.Secret { + t.Errorf("expected %v but got %v", secret.Secret, u.AuthSSLCert.Secret) + } + if u.ProxySSLCA.ConfigMap != configMap.ConfigMap { + t.Errorf("expected %v but got %v", secret.Secret, u.AuthSSLCert.Secret) + } if u.Ciphers != proxySslCiphers { t.Errorf("expected %v but got %v", proxySslCiphers, u.Ciphers) } @@ -179,6 +220,8 @@ func TestInvalidAnnotations(t *testing.T) { // Invalid optional Annotations data[parser.GetAnnotationWithPrefix("proxy-ssl-secret")] = defaultDemoSecret + data[parser.GetAnnotationWithPrefix("proxy-ssl-client-secret")] = defaultDemoSecret + data[parser.GetAnnotationWithPrefix("proxy-ssl-ca-configmap")] = defaultDemoConfigMap data[parser.GetAnnotationWithPrefix("proxy-ssl-protocols")] = "TLSv111 SSLv1" data[parser.GetAnnotationWithPrefix("proxy-ssl-server-name")] = sslServerName data[parser.GetAnnotationWithPrefix("proxy-ssl-session-reuse")] = sslServerName @@ -237,6 +280,9 @@ func TestEquals(t *testing.T) { t.Errorf("Expected false") } cfg2.AuthSSLCert = sslCert1 + // TODO: Different client certs + + // TODO: Different CAs // Different Ciphers cfg1.Ciphers = "DEFAULT" diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go index 652a80e498..a3df6bf245 100644 --- a/internal/ingress/controller/controller.go +++ b/internal/ingress/controller/controller.go @@ -749,9 +749,9 @@ func (n *NGINXController) getBackendServers(ingresses []*ingress.Ingress) ([]*in } if !n.store.GetBackendConfiguration().ProxySSLLocationOnly { - if server.ProxySSL.CAFileName == "" { + if server.ProxySSL.CAFileName == "" && server.ProxySSL.ProxySSLCA.CAFileName == "" { server.ProxySSL = anns.ProxySSL - if server.ProxySSL.Secret != "" && server.ProxySSL.CAFileName == "" { + if (server.ProxySSL.Secret != "" && server.ProxySSL.CAFileName == "") && (server.ProxySSL.ProxySSLCA.ConfigMap != "" && server.ProxySSL.ProxySSLCA.CAFileName == "") { klog.V(3).Infof("Secret %q has no 'ca.crt' key, client cert authentication disabled for Ingress %q", server.ProxySSL.Secret, ingKey) } diff --git a/internal/ingress/controller/controller_test.go b/internal/ingress/controller/controller_test.go index 9d3fea4708..63deefaaa8 100644 --- a/internal/ingress/controller/controller_test.go +++ b/internal/ingress/controller/controller_test.go @@ -121,6 +121,14 @@ func (fakeIngressStore) GetAuthCertificate(string) (*resolver.AuthSSLCert, error return nil, fmt.Errorf("test error") } +func (fakeIngressStore) GetSSLClientCert(string) (*resolver.SSLClientCert, error) { + return nil, fmt.Errorf("test error") +} + +func (fakeIngressStore) GetSSLCA(string) (*resolver.SSLCA, error) { + return nil, fmt.Errorf("test error") +} + func (fakeIngressStore) GetDefaultBackend() defaults.Backend { return defaults.Backend{} } diff --git a/internal/ingress/controller/store/backend_ssl.go b/internal/ingress/controller/store/backend_ssl.go index 81b508cd23..14db52aa40 100644 --- a/internal/ingress/controller/store/backend_ssl.go +++ b/internal/ingress/controller/store/backend_ssl.go @@ -36,12 +36,29 @@ import ( // syncSecret synchronizes the content of a TLS Secret (certificate(s), secret // key) with the filesystem. The resulting files can be used by NGINX. func (s *k8sStore) syncSecret(key string) { + s.syncResource(key, "Secret", s.getPemCertificate) +} + +// syncSecret synchronizes the content of a client TLS Secret (certificate, secret +// key) with the filesystem. The resulting files can be used by NGINX. +func (s *k8sStore) syncClientCertSecret(key string) { + s.syncResource(key, "Secret", s.getClientPemCertificate) +} + +// syncCAConfigMap synchronizes the content of a ConfigMap including CAs +// with the filesystem. The resulting file can be used by NGINX. +func (s *k8sStore) syncCAConfigMap(key string) { + s.syncResource(key, "ConfigMap", s.getCAPEMCertificate) +} + +// syncResource synchronizes the content of a K8s resource (Secret, ConfigMap...) +func (s *k8sStore) syncResource(key, resourceKind string, getPermCert func(string) (*ingress.SSLCert, error)) { s.syncSecretMu.Lock() defer s.syncSecretMu.Unlock() - klog.V(3).InfoS("Syncing Secret", "name", key) + klog.V(3).InfoS(fmt.Sprintf("Syncing %s", resourceKind), "name", key) - cert, err := s.getPemCertificate(key) + cert, err := getPermCert(key) if err != nil { if !isErrSecretForAuth(err) { klog.Warningf("Error obtaining X.509 certificate: %v", err) @@ -56,7 +73,7 @@ func (s *k8sStore) syncSecret(key string) { // no need to update return } - klog.InfoS("Updating secret in local store", "name", key) + klog.InfoS("Updating cert in local store", "name", key) s.sslStore.Update(key, cert) // this update must trigger an update // (like an update event from a change in Ingress) @@ -64,7 +81,7 @@ func (s *k8sStore) syncSecret(key string) { return } - klog.InfoS("Adding secret to local store", "name", key) + klog.InfoS("Adding cert to local store", "name", key) s.sslStore.Add(key, cert) // this update must trigger an update // (like an update event from a change in Ingress) @@ -146,27 +163,10 @@ func (s *k8sStore) getPemCertificate(secretName string) (*ingress.SSLCert, error klog.V(3).InfoS(msg) case len(ca) > 0: - sslCert, err = ssl.CreateCACert(ca) + sslCert, err = createCASSLCert(secretName, "Secret", string(secret.UID), ca, crl) if err != nil { - return nil, fmt.Errorf("unexpected error creating SSL Cert: %v", err) + return nil, err } - - err = ssl.ConfigureCACert(nsSecName, ca, sslCert) - if err != nil { - return nil, fmt.Errorf("error configuring CA certificate: %v", err) - } - - sslCert.CASHA = file.SHA1(sslCert.CAFileName) - - if len(crl) > 0 { - err = ssl.ConfigureCRL(nsSecName, crl, sslCert) - if err != nil { - return nil, err - } - } - // makes this secret in 'syncSecret' to be used for Certificate Authentication - // this does not enable Certificate Authentication - klog.V(3).InfoS("Configuring Secret for TLS authentication", "secret", secretName) default: if auth != nil { return nil, ErrSecretForAuth @@ -191,6 +191,88 @@ func (s *k8sStore) getPemCertificate(secretName string) (*ingress.SSLCert, error return sslCert, nil } +// getClientPemCertificate receives a secret, and creates an ingress.SSLCert as return. +// It parses the secret and verifies they keypair. +func (s *k8sStore) getClientPemCertificate(secretName string) (*ingress.SSLCert, error) { + secret, err := s.listers.Secret.ByKey(secretName) + if err != nil { + return nil, err + } + + cert, okcert := secret.Data[apiv1.TLSCertKey] + if !okcert || cert == nil { + return nil, fmt.Errorf("key 'tls.crt' missing from Secret %q", secretName) + } + + key, okkey := secret.Data[apiv1.TLSPrivateKeyKey] + if !okkey || key == nil { + return nil, fmt.Errorf("key 'tls.key' missing from Secret %q", secretName) + } + + // namespace/secretName -> namespace-secretName + nsSecName := strings.ReplaceAll(secretName, "/", "-") + + sslCert, err := ssl.CreateSSLCert(cert, key, string(secret.UID)) + if err != nil { + return nil, fmt.Errorf("unexpected error creating SSL Cert: %v", err) + } + + path, err := ssl.StoreSSLCertOnDisk(nsSecName, sslCert) + if err != nil { + return nil, fmt.Errorf("error while storing certificate and key: %v", err) + } + sslCert.PemFileName = path + + klog.V(3).InfoS(fmt.Sprintf("Configuring Secret %q for TLS encryption (CN: %v)", secretName, sslCert.CN)) + + sslCert.Name = secret.Name + sslCert.Namespace = secret.Namespace + + return sslCert, nil +} + +// getCAPEMCertificate receives a configMap, and creates an ingress.SSLCert as return. +// It parses the configMap +func (s *k8sStore) getCAPEMCertificate(configMapName string) (*ingress.SSLCert, error) { + configmap, err := s.listers.ConfigMap.ByKey(configMapName) + if err != nil { + return nil, err + } + + ca := configmap.Data["ca.crt"] + crl := configmap.Data["ca.crl"] + + return createCASSLCert(configMapName, "ConfigMap", string(configmap.UID), []byte(ca), []byte(crl)) +} + +func createCASSLCert(resourceName, resourceKind, resourceUID string, ca, crl []byte) (*ingress.SSLCert, error) { + nsName := strings.ReplaceAll(resourceName, "/", "-") + + sslCert, err := ssl.CreateCACert(ca) + if err != nil { + return nil, fmt.Errorf("unexpected error creating SSL Cert: %v", err) + } + + sslCert.UID = resourceUID + + err = ssl.ConfigureCACert(nsName, ca, sslCert) + if err != nil { + return nil, fmt.Errorf("error configuring CA certificate: %v", err) + } + + sslCert.CASHA = file.SHA1(sslCert.CAFileName) + + if len(crl) > 0 { + err = ssl.ConfigureCRL(nsName, crl, sslCert) + if err != nil { + return nil, err + } + } + klog.V(3).InfoS("Configuring certs for TLS authentication", resourceKind, resourceName) + + return sslCert, nil +} + // sendDummyEvent sends a dummy event to trigger an update // This is used in when a secret change func (s *k8sStore) sendDummyEvent() { diff --git a/internal/ingress/controller/store/store.go b/internal/ingress/controller/store/store.go index d4bd6136f8..8c575801c5 100644 --- a/internal/ingress/controller/store/store.go +++ b/internal/ingress/controller/store/store.go @@ -22,6 +22,7 @@ import ( "fmt" "os" "reflect" + "slices" "sort" "strings" "sync" @@ -36,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/labels" k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" @@ -98,6 +100,12 @@ type Storer interface { // ca.crt: contains the certificate chain used for authentication GetAuthCertificate(string) (*resolver.AuthSSLCert, error) + // GetSSLClientCert resolves a given secret name into an SSL certificate. + GetSSLClientCert(string) (*resolver.SSLClientCert, error) + + // GetSSLCA resolves a given configMap name into an SSL CA. + GetSSLCA(string) (*resolver.SSLCA, error) + // GetDefaultBackend returns the default backend configuration GetDefaultBackend() defaults.Backend @@ -230,6 +238,14 @@ type k8sStore struct { // secret in the annotations. secretIngressMap ObjectRefMap + // clientCertSecretIngressMap contains information about which ingress references a + // client cert secret in the annotations. + clientCertSecretIngressMap ObjectRefMap + + // configmapIngressMap contains information about which ingress references a + // configMap in the annotations. + caConfigMapIngressMap ObjectRefMap + // updateCh updateCh *channels.RingChannel @@ -260,15 +276,17 @@ func New( disableSyncEvents bool, ) Storer { store := &k8sStore{ - informers: &Informer{}, - listers: &Lister{}, - sslStore: NewSSLCertTracker(), - updateCh: updateCh, - backendConfig: ngx_config.NewDefault(), - syncSecretMu: &sync.Mutex{}, - backendConfigMu: &sync.RWMutex{}, - secretIngressMap: NewObjectRefMap(), - defaultSSLCertificate: defaultSSLCertificate, + informers: &Informer{}, + listers: &Lister{}, + sslStore: NewSSLCertTracker(), + updateCh: updateCh, + backendConfig: ngx_config.NewDefault(), + syncSecretMu: &sync.Mutex{}, + backendConfigMu: &sync.RWMutex{}, + secretIngressMap: NewObjectRefMap(), + clientCertSecretIngressMap: NewObjectRefMap(), + caConfigMapIngressMap: NewObjectRefMap(), + defaultSSLCertificate: defaultSSLCertificate, } eventBroadcaster := record.NewBroadcaster() @@ -458,6 +476,10 @@ func New( store.syncIngress(ing) store.updateSecretIngressMap(ing) store.syncSecrets(ing) + store.updateClientCertSecretIngressMap(ing) + store.syncClientCertSecrets(ing) + store.updateCAConfigMapIngressMap(ing) + store.syncCAConfigMaps(ing) updateCh.In() <- Event{ Type: CreateEvent, @@ -515,6 +537,10 @@ func New( store.syncIngress(curIng) store.updateSecretIngressMap(curIng) store.syncSecrets(curIng) + store.updateClientCertSecretIngressMap(curIng) + store.syncClientCertSecrets(curIng) + store.updateCAConfigMapIngressMap(curIng) + store.syncCAConfigMaps(curIng) updateCh.In() <- Event{ Type: UpdateEvent, @@ -609,9 +635,9 @@ func New( store.syncSecret(store.defaultSSLCertificate) } - // find references in ingresses and update local ssl certs + // find references in ingresses for SSL secret and update local ssl certs if ings := store.secretIngressMap.Reference(key); len(ings) > 0 { - klog.InfoS("Secret was added and it is used in ingress annotations. Parsing", "secret", key) + klog.InfoS("SSL Secret was added and it is used in ingress annotations. Parsing", "secret", key) for _, ingKey := range ings { ing, err := store.getIngress(ingKey) if err != nil { @@ -626,6 +652,24 @@ func New( Obj: obj, } } + + // find references in ingresses for client cert secret and update local ssl certs + if ings := store.clientCertSecretIngressMap.Reference(key); len(ings) > 0 { + klog.InfoS("Client cert Secret was added and it is used in ingress annotations. Parsing", "secret", key) + for _, ingKey := range ings { + ing, err := store.getIngress(ingKey) + if err != nil { + klog.Errorf("could not find Ingress %v in local store", ingKey) + continue + } + store.syncIngress(ing) + store.syncClientCertSecret(key) + } + updateCh.In() <- Event{ + Type: CreateEvent, + Obj: obj, + } + } }, UpdateFunc: func(old, cur interface{}) { if !reflect.DeepEqual(old, cur) { @@ -643,9 +687,9 @@ func New( store.syncSecret(store.defaultSSLCertificate) } - // find references in ingresses and update local ssl certs + // find references in ingresses for SSL secret and update local ssl certs if ings := store.secretIngressMap.Reference(key); len(ings) > 0 { - klog.InfoS("secret was updated and it is used in ingress annotations. Parsing", "secret", key) + klog.InfoS("SSL secret was updated and it is used in ingress annotations. Parsing", "secret", key) for _, ingKey := range ings { ing, err := store.getIngress(ingKey) if err != nil { @@ -660,6 +704,24 @@ func New( Obj: cur, } } + + // find references in ingresses for SSL client secret and update local ssl certs + if ings := store.clientCertSecretIngressMap.Reference(key); len(ings) > 0 { + klog.InfoS("client cert secret was updated and it is used in ingress annotations. Parsing", "secret", key) + for _, ingKey := range ings { + ing, err := store.getIngress(ingKey) + if err != nil { + klog.ErrorS(err, "could not find Ingress in local store", "ingress", ingKey) + continue + } + store.syncClientCertSecret(key) + store.syncIngress(ing) + } + updateCh.In() <- Event{ + Type: UpdateEvent, + Obj: cur, + } + } } }, DeleteFunc: func(obj interface{}) { @@ -686,9 +748,11 @@ func New( key := k8s.MetaNamespaceKey(sec) // find references in ingresses - if ings := store.secretIngressMap.Reference(key); len(ings) > 0 { + ings := sets.New(store.secretIngressMap.Reference(key)...) + ings.Insert(store.clientCertSecretIngressMap.Reference(key)...) + if ings.Len() > 0 { klog.InfoS("secret was deleted and it is used in ingress annotations. Parsing", "secret", key) - for _, ingKey := range ings { + for _, ingKey := range ings.UnsortedList() { ing, err := store.getIngress(ingKey) if err != nil { klog.Errorf("could not find Ingress %v in local store", ingKey) @@ -740,18 +804,20 @@ func New( return name == configmap || name == tcp || name == udp } - handleCfgMapEvent := func(key string, cfgMap *corev1.ConfigMap, eventName string) { + handleCfgMapEvent := func(key string, cfgMap *corev1.ConfigMap, event EventType) { // updates to configuration configmaps can trigger an update triggerUpdate := false if changeTriggerUpdate(key) { triggerUpdate = true - recorder.Eventf(cfgMap, corev1.EventTypeNormal, eventName, fmt.Sprintf("ConfigMap %v", key)) + recorder.Eventf(cfgMap, corev1.EventTypeNormal, string(event), fmt.Sprintf("ConfigMap %v", key)) if key == configmap { store.setConfig(cfgMap) } } ings := store.listers.IngressWithAnnotation.List() + ingRefCM := store.caConfigMapIngressMap.Reference(key) + for _, ingKey := range ings { key := k8s.MetaNamespaceKey(ingKey) ing, err := store.getIngress(key) @@ -760,6 +826,19 @@ func New( continue } + if slices.Contains(ingRefCM, key) { + klog.InfoS("ca config map was updated and it is used in ingress annotations. Parsing", "configmap", key) + cmKey := k8s.MetaNamespaceKey(configmap) + + store.syncCAConfigMap(cmKey) + store.syncIngress(ing) + updateCh.In() <- Event{ + Type: event, + Obj: cfgMap, + } + continue + } + if parser.AnnotationsReferencesConfigmap(ing) { store.syncIngress(ing) continue @@ -799,6 +878,48 @@ func New( key := k8s.MetaNamespaceKey(cfgMap) handleCfgMapEvent(key, cfgMap, "UPDATE") }, + DeleteFunc: func(obj interface{}) { + cfgMap, ok := obj.(*corev1.ConfigMap) + + if !ok { + // If we reached here it means the configmap was deleted but its final state is unrecorded. + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + return + } + + cfgMap, ok = tombstone.Obj.(*corev1.ConfigMap) + if !ok { + return + } + } + + if !watchedNamespace(cfgMap.Namespace) { + return + } + + store.sslStore.Delete(k8s.MetaNamespaceKey(cfgMap)) + + key := k8s.MetaNamespaceKey(cfgMap) + + // find references in ingresses + if ings := store.caConfigMapIngressMap.Reference(key); len(ings) > 0 { + klog.InfoS("configmap was deleted and it is used in ingress annotations. Parsing", "configMap", key) + for _, ingKey := range ings { + ing, err := store.getIngress(ingKey) + if err != nil { + klog.Errorf("could not find Ingress %v in local store", ingKey) + continue + } + store.syncIngress(ing) + } + + updateCh.In() <- Event{ + Type: DeleteEvent, + Obj: obj, + } + } + }, } serviceHandler := cache.ResourceEventHandlerFuncs{ @@ -999,6 +1120,53 @@ func (s *k8sStore) updateSecretIngressMap(ing *networkingv1.Ingress) { s.secretIngressMap.Insert(key, refSecrets...) } +// updateSecretIngressMap takes an Ingress and updates all Secret objects it +// references in secretIngressMap. +func (s *k8sStore) updateClientCertSecretIngressMap(ing *networkingv1.Ingress) { + key := k8s.MetaNamespaceKey(ing) + klog.V(3).Infof("updating references to client cert secrets for ingress %v", key) + + // delete all existing references first + s.clientCertSecretIngressMap.Delete(key) + + secConfig := s.GetSecurityConfiguration().AllowCrossNamespaceResources + var refClientCertSecrets []string + secrKey, err := objectRefAnnotationNsKey("proxy-ssl-client-secret", ing, secConfig) + if err != nil && !errors.IsMissingAnnotations(err) { + klog.Errorf("error reading client secret reference in annotation %q: %s", "proxy-ssl-client-secret", err) + } + if secrKey != "" { + refClientCertSecrets = append(refClientCertSecrets, secrKey) + } + + // populate map with all secret references + s.clientCertSecretIngressMap.Insert(key, refClientCertSecrets...) +} + +// updateConfigMapIngressMap takes an Ingress and updates all ConfigMap objects it +// references in configMapIngressMap. +func (s *k8sStore) updateCAConfigMapIngressMap(ing *networkingv1.Ingress) { + key := k8s.MetaNamespaceKey(ing) + klog.V(3).Infof("updating references to configmaps for ingress %v", key) + + // delete all existing references first + s.caConfigMapIngressMap.Delete(key) + + var refCACms []string + + secConfig := s.GetSecurityConfiguration().AllowCrossNamespaceResources + cmKey, err := objectRefAnnotationNsKey("proxy-ssl-ca-configmap", ing, secConfig) + if err != nil && !errors.IsMissingAnnotations(err) { + klog.Errorf("error reading ca configmap reference in annotation %q: %s", "proxy-ssl-ca-configmap", err) + } + if cmKey != "" { + refCACms = append(refCACms, cmKey) + } + + // populate map with all secret references + s.caConfigMapIngressMap.Insert(key, refCACms...) +} + // objectRefAnnotationNsKey returns an object reference formatted as a // 'namespace/name' key from the given annotation name. func objectRefAnnotationNsKey(ann string, ing *networkingv1.Ingress, allowCrossNamespace bool) (string, error) { @@ -1031,6 +1199,24 @@ func (s *k8sStore) syncSecrets(ing *networkingv1.Ingress) { } } +// syncClientCertSecrets synchronizes data from client cert Secrets referenced by the given +// Ingress with the local store and file system. +func (s *k8sStore) syncClientCertSecrets(ing *networkingv1.Ingress) { + key := k8s.MetaNamespaceKey(ing) + for _, secrKey := range s.clientCertSecretIngressMap.ReferencedBy(key) { + s.syncClientCertSecret(secrKey) + } +} + +// syncCAConfigMaps synchronizes data from CA configmaps referenced by the given +// Ingress with the local store and file system. +func (s *k8sStore) syncCAConfigMaps(ing *networkingv1.Ingress) { + key := k8s.MetaNamespaceKey(ing) + for _, cmKey := range s.caConfigMapIngressMap.ReferencedBy(key) { + s.syncCAConfigMap(cmKey) + } +} + // GetSecret returns the Secret matching key. func (s *k8sStore) GetSecret(key string) (*corev1.Secret, error) { return s.listers.Secret.ByKey(key) @@ -1156,6 +1342,43 @@ func (s *k8sStore) GetAuthCertificate(name string) (*resolver.AuthSSLCert, error }, nil } +// GetSSLClientCert is used by the proxy-ssl annotations to get a cert from a secret +func (s *k8sStore) GetSSLClientCert(name string) (*resolver.SSLClientCert, error) { + if _, err := s.GetLocalSSLCert(name); err != nil { + s.syncClientCertSecret(name) + } + + cert, err := s.GetLocalSSLCert(name) + if err != nil { + return nil, err + } + + return &resolver.SSLClientCert{ + Secret: name, + PemFileName: cert.PemFileName, + }, nil +} + +// GetSSLCA is used by the proxy-ssl annotations to get a ca from a configmap +func (s *k8sStore) GetSSLCA(configMapName string) (*resolver.SSLCA, error) { + if _, err := s.GetLocalSSLCert(configMapName); err != nil { + s.syncCAConfigMap(configMapName) + } + + cert, err := s.GetLocalSSLCert(configMapName) + if err != nil { + return nil, err + } + + return &resolver.SSLCA{ + ConfigMap: configMapName, + CAFileName: cert.CAFileName, + CASHA: cert.CASHA, + CRLFileName: cert.CRLFileName, + CRLSHA: cert.CRLSHA, + }, nil +} + func (s *k8sStore) writeSSLSessionTicketKey(cmap *corev1.ConfigMap, fileName string) { ticketString := ngx_template.ReadConfig(cmap.Data).SSLSessionTicketKey s.backendConfig.SSLSessionTicketKey = "" diff --git a/internal/ingress/controller/template/template.go b/internal/ingress/controller/template/template.go index ed052e4ecf..6012689545 100644 --- a/internal/ingress/controller/template/template.go +++ b/internal/ingress/controller/template/template.go @@ -784,7 +784,7 @@ func filterRateLimits(input interface{}) []ratelimit.Config { servers, ok := input.([]*ingress.Server) if !ok { - klog.Errorf("expected a '[]ratelimit.RateLimit' type but %T was returned", input) + klog.Errorf("expected a '[]*ingress.Server' type but %T was returned", input) return ratelimits } for _, server := range servers { diff --git a/internal/ingress/resolver/main.go b/internal/ingress/resolver/main.go index 259f44e49c..ca968ae719 100644 --- a/internal/ingress/resolver/main.go +++ b/internal/ingress/resolver/main.go @@ -42,6 +42,12 @@ type Resolver interface { // ca.crl: contains the revocation list used for authentication GetAuthCertificate(string) (*AuthSSLCert, error) + // GetSSLClientCert resolves a given secret name into an SSL certificate. + GetSSLClientCert(string) (*SSLClientCert, error) + + // GetSSLCA resolves a given configMap name into an SSL CA. + GetSSLCA(string) (*SSLCA, error) + // GetService searches for services containing the namespace and name using the character / GetService(string) (*apiv1.Service, error) } @@ -91,3 +97,70 @@ func (asslc1 *AuthSSLCert) Equal(assl2 *AuthSSLCert) bool { return true } + +// SSLClientCert contains the clients certificate information +type SSLClientCert struct { + // Secret contains the name of the secret this was fetched from + Secret string `json:"secret"` + // PemFileName contains the path to the secrets 'tls.crt' and 'tls.key' + PemFileName string `json:"pemFilename"` +} + +// Equal tests for equality between two SSLClientCert types +func (sslcc1 *SSLClientCert) Equal(sslcc2 *SSLClientCert) bool { + if sslcc1 == sslcc2 { + return true + } + if sslcc1 == nil || sslcc2 == nil { + return false + } + + if sslcc1.Secret != sslcc2.Secret { + return false + } + + return true +} + +// SSLCA contains the CAs used to validate client certificates +type SSLCA struct { + // ConfigMap contains the name of the configMap this was fetched from + ConfigMap string `json:"configmap"` + // CAFileName contains the path to the secrets 'ca.crt' + CAFileName string `json:"caFilename"` + // CASHA contains the SHA1 hash of the 'ca.crt' + CASHA string `json:"caSha"` + // CRLFileName contains the path to the secrets 'ca.crl' + CRLFileName string `json:"crlFileName"` + // CRLSHA contains the SHA1 hash of the 'ca.crl' file + CRLSHA string `json:"crlSha"` +} + +// Equal tests for equality between two SSLCA types +func (sslc1 *SSLCA) Equal(sslc2 *SSLCA) bool { + if sslc1 == sslc2 { + return true + } + if sslc1 == nil || sslc2 == nil { + return false + } + + if sslc1.ConfigMap != sslc2.ConfigMap { + return false + } + if sslc1.CAFileName != sslc2.CAFileName { + return false + } + if sslc1.CASHA != sslc2.CASHA { + return false + } + + if sslc1.CRLFileName != sslc2.CRLFileName { + return false + } + if sslc1.CRLSHA != sslc2.CRLSHA { + return false + } + + return true +} diff --git a/internal/ingress/resolver/main_test.go b/internal/ingress/resolver/main_test.go index 044351310d..dce2749145 100644 --- a/internal/ingress/resolver/main_test.go +++ b/internal/ingress/resolver/main_test.go @@ -58,3 +58,5 @@ func TestAuthSSLCertEqual(t *testing.T) { } } } + +// TODO : implement tests for GetSSLClientCert and GetSSLCA diff --git a/internal/ingress/resolver/mock.go b/internal/ingress/resolver/mock.go index 3abfe7eda3..ec324c7ae7 100644 --- a/internal/ingress/resolver/mock.go +++ b/internal/ingress/resolver/mock.go @@ -60,6 +60,16 @@ func (m Mock) GetAuthCertificate(string) (*AuthSSLCert, error) { return nil, nil } +// GetSSLClientCert resolves a given secret name into an SSL certificate. +func (m Mock) GetSSLClientCert(string) (*SSLClientCert, error) { + return nil, nil +} + +// GetSSLCA resolves a given configMap name into an SSL CA. +func (m Mock) GetSSLCA(string) (*SSLCA, error) { + return nil, nil +} + // GetService searches for services containing the namespace and name using the character / func (m Mock) GetService(string) (*apiv1.Service, error) { return nil, nil diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index 6b8e750b06..002044e521 100644 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -912,9 +912,14 @@ stream { {{ end }} {{ end }} - {{ if not (empty $server.ProxySSL.CAFileName) }} + {{ if or (not (empty $server.ProxySSL.ProxySSLCA.CAFileName)) (not (empty $server.ProxySSL.CAFileName)) }} + {{ if not (empty $server.ProxySSL.ProxySSLCA.CAFileName) }} + # PEM sha: {{ $server.ProxySSL.ProxySSLCA.CASHA }} + proxy_ssl_trusted_certificate {{ $server.ProxySSL.ProxySSLCA.CAFileName }}; + {{ else if not (empty $server.ProxySSL.CAFileName) }} # PEM sha: {{ $server.ProxySSL.CASHA }} proxy_ssl_trusted_certificate {{ $server.ProxySSL.CAFileName }}; + {{ end }} proxy_ssl_ciphers {{ $server.ProxySSL.Ciphers }}; proxy_ssl_protocols {{ $server.ProxySSL.Protocols }}; proxy_ssl_verify {{ $server.ProxySSL.Verify }}; @@ -925,7 +930,10 @@ stream { {{ end }} {{ end }} - {{ if not (empty $server.ProxySSL.PemFileName) }} + {{ if not (empty $server.ProxySSL.ProxySSLClientCert.PemFileName) }} + proxy_ssl_certificate {{ $server.ProxySSL.ProxySSLClientCert.PemFileName }}; + proxy_ssl_certificate_key {{ $server.ProxySSL.ProxySSLClientCert.PemFileName }}; + {{ else if not (empty $server.ProxySSL.PemFileName) }} proxy_ssl_certificate {{ $server.ProxySSL.PemFileName }}; proxy_ssl_certificate_key {{ $server.ProxySSL.PemFileName }}; {{ end }} @@ -1386,9 +1394,14 @@ stream { # Location denied. Reason: {{ $location.Denied | quote }} return 503; {{ end }} - {{ if not (empty $location.ProxySSL.CAFileName) }} + {{ if or (not (empty $location.ProxySSL.ProxySSLCA.CAFileName)) (not (empty $location.ProxySSL.CAFileName)) }} + {{ if not (empty $location.ProxySSL.ProxySSLCA.CAFileName) }} + # PEM sha: {{ $location.ProxySSL.ProxySSLCA.CASHA }} + proxy_ssl_trusted_certificate {{ $location.ProxySSL.ProxySSLCA.CAFileName }}; + {{ else if not (empty $location.ProxySSL.CAFileName) }} # PEM sha: {{ $location.ProxySSL.CASHA }} proxy_ssl_trusted_certificate {{ $location.ProxySSL.CAFileName }}; + {{ end }} proxy_ssl_ciphers {{ $location.ProxySSL.Ciphers }}; proxy_ssl_protocols {{ $location.ProxySSL.Protocols }}; proxy_ssl_verify {{ $location.ProxySSL.Verify }}; @@ -1402,7 +1415,10 @@ stream { proxy_ssl_server_name {{ $location.ProxySSL.ProxySSLServerName }}; {{ end }} - {{ if not (empty $location.ProxySSL.PemFileName) }} + {{ if not (empty $location.ProxySSL.ProxySSLClientCert.PemFileName) }} + proxy_ssl_certificate {{ $location.ProxySSL.ProxySSLClientCert.PemFileName }}; + proxy_ssl_certificate_key {{ $location.ProxySSL.ProxySSLClientCert.PemFileName }}; + {{ else if not (empty $location.ProxySSL.PemFileName) }} proxy_ssl_certificate {{ $location.ProxySSL.PemFileName }}; proxy_ssl_certificate_key {{ $location.ProxySSL.PemFileName }}; {{ end }}