From 2e274b306a9876c71791492ce38fb9ccd330c2ad Mon Sep 17 00:00:00 2001 From: Kaushal M Date: Fri, 12 Apr 2019 21:39:08 +0530 Subject: [PATCH] ceph: Use aggregated ClusterRoles The ClusterRoles in the manifests (and helm chart) for the ceph provider have been updated to use [aggregated ClusterRoles][1]. All ClusterRoles have been split into 2 ClusterRoles. An aggregated ClusterRole without any rules, named the same, but with an aggregationRule that matches labels in the format 'rbac.ceph.rook.io/aggregate-to-: "true"'. A second ClusterRole which contains the existing rules named '-rules', with label 'rbac.ceph.rook.io/aggregate-to-: "true"'. These two ClusterRoles give the same behaviour as the previous ClusterRole. Additional rules to a ClusterRole can now be provided by creating a new ClusterRole with the correct label. This should help make updates simpler. Fixes #2634. [1]: https://kubernetes.io/docs/reference/access-authn-authz/rbac/#aggregated-clusterroles Signed-off-by: Kaushal M --- .../rook-ceph/templates/clusterrole.yaml | 83 +++++++++++++ cluster/examples/kubernetes/ceph/common.yaml | 55 +++++++++ .../csi/rbac/cephfs/csi-nodeplugin-rbac.yaml | 12 ++ .../csi/rbac/cephfs/csi-provisioner-rbac.yaml | 12 ++ .../csi/rbac/rbd/csi-nodeplugin-rbac.yaml | 12 ++ .../csi/rbac/rbd/csi-provisioner-rbac.yaml | 12 ++ .../ceph/monitoring/prometheus.yaml | 13 ++ tests/framework/installer/ceph_installer.go | 8 ++ tests/framework/installer/ceph_manifests.go | 115 ++++++++++++++++++ 9 files changed, 322 insertions(+) diff --git a/cluster/charts/rook-ceph/templates/clusterrole.yaml b/cluster/charts/rook-ceph/templates/clusterrole.yaml index da146f5b1773..9869fc253dab 100644 --- a/cluster/charts/rook-ceph/templates/clusterrole.yaml +++ b/cluster/charts/rook-ceph/templates/clusterrole.yaml @@ -7,6 +7,20 @@ metadata: labels: operator: rook storage-backend: ceph +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.rook.ceph.io/aggregate-to-rook-ceph-cluster-mgmt: "true" +rules: [] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-cluster-mgmt-rules + labels: + operator: rook + storage-backend: ceph + rbac.rook.ceph.io/aggregate-to-rook-ceph-cluster-mgmt: "true" rules: - apiGroups: - "" @@ -46,6 +60,20 @@ metadata: labels: operator: rook storage-backend: ceph +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" +rules: [] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-global-rules + labels: + operator: rook + storage-backend: ceph + rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" rules: - apiGroups: - "" @@ -115,6 +143,20 @@ metadata: labels: operator: rook storage-backend: ceph +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-mgr-cluster-rules + labels: + operator: rook + storage-backend: ceph + rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true" rules: - apiGroups: - "" @@ -132,6 +174,18 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-mgr-system +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-system: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-mgr-system-rules + labels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-system: "true" rules: - apiGroups: - "" @@ -150,6 +204,20 @@ metadata: labels: operator: rook storage-backend: ceph +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-agent-mount: "true" +rules: [] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-agent-mount-rules + labels: + operator: rook + storage-backend: ceph + rbac.ceph.rook.io/aggregate-to-rook-ceph-agent-mount: "true" rules: - apiGroups: - "" @@ -168,6 +236,21 @@ metadata: operator: rook storage-backend: ceph chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-system-psp-user: "true" +rules: [] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-system-psp-user-rules + labels: + operator: rook + storage-backend: ceph + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + rbac.ceph.rook.io/aggregate-to-rook-ceph-system-psp-user: "true" rules: - apiGroups: - apps diff --git a/cluster/examples/kubernetes/ceph/common.yaml b/cluster/examples/kubernetes/ceph/common.yaml index 215071d34f71..fbf5fcf9a61b 100644 --- a/cluster/examples/kubernetes/ceph/common.yaml +++ b/cluster/examples/kubernetes/ceph/common.yaml @@ -206,6 +206,20 @@ metadata: labels: operator: rook storage-backend: ceph +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-cluster-mgmt: "true" +rules: [] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-cluster-mgmt-rules + labels: + operator: rook + storage-backend: ceph + rbac.ceph.rook.io/aggregate-to-rook-ceph-cluster-mgmt: "true" rules: - apiGroups: - "" @@ -282,6 +296,20 @@ metadata: labels: operator: rook storage-backend: ceph +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" +rules: [] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-global-rules + labels: + operator: rook + storage-backend: ceph + rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" rules: - apiGroups: - "" @@ -351,6 +379,20 @@ metadata: labels: operator: rook storage-backend: ceph +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-mgr-cluster-rules + labels: + operator: rook + storage-backend: ceph + rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true" rules: - apiGroups: - "" @@ -444,6 +486,19 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-mgr-system namespace: rook-ceph +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-system: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-mgr-system-rules + namespace: rook-ceph + labels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-system: "true" rules: - apiGroups: - "" diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-nodeplugin-rbac.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-nodeplugin-rbac.yaml index 67e24ba43df7..5fb0bb1228dd 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-nodeplugin-rbac.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-nodeplugin-rbac.yaml @@ -8,6 +8,18 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cephfs-csi-nodeplugin +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-cephfs-csi-nodeplugin: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-nodeplugin-rules + labels: + rbac.ceph.rook.io/aggregate-to-cephfs-csi-nodeplugin: "true" rules: - apiGroups: [""] resources: ["nodes"] diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-rbac.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-rbac.yaml index 5c099f7bbaaf..fdcc18bf79e1 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-rbac.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-rbac.yaml @@ -8,6 +8,18 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cephfs-external-provisioner-runner +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-external-provisioner-runner-rules + labels: + rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner: "true" rules: - apiGroups: [""] resources: ["secrets"] diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-nodeplugin-rbac.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-nodeplugin-rbac.yaml index 46791823df5b..d37d0cc86488 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-nodeplugin-rbac.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-nodeplugin-rbac.yaml @@ -9,6 +9,18 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-csi-nodeplugin +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rbd-csi-nodeplugin: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin-rules + labels: + rbac.ceph.rook.io/aggregate-to-rbd-csi-nodeplugin: "true" rules: - apiGroups: [""] resources: ["nodes"] diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-rbac.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-rbac.yaml index 5969b5d066d6..028d7bd8243b 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-rbac.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-rbac.yaml @@ -9,6 +9,18 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-external-provisioner-runner +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-external-provisioner-runner-rules + labels: + rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner: "true" rules: - apiGroups: [""] resources: ["secrets"] diff --git a/cluster/examples/kubernetes/ceph/monitoring/prometheus.yaml b/cluster/examples/kubernetes/ceph/monitoring/prometheus.yaml index cc7b1af1f2f0..49a70112e1f1 100644 --- a/cluster/examples/kubernetes/ceph/monitoring/prometheus.yaml +++ b/cluster/examples/kubernetes/ceph/monitoring/prometheus.yaml @@ -9,6 +9,19 @@ kind: ClusterRole metadata: name: prometheus namespace: rook-ceph +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-prometheus: "true" +rules: [] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: prometheus-rules + namespace: rook-ceph + labels: + rbac.ceph.rook.io/aggregate-to-prometheus: "true" rules: - apiGroups: [""] resources: diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 60a5075e47ca..2d225be87ee9 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -407,20 +407,28 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, name h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-ceph-mgr-cluster", nil) h.k8shelper.Clientset.CoreV1().ServiceAccounts(systemNamespace).Delete("rook-ceph-system", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-cluster-mgmt", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-cluster-mgmt-rules", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-mgr-cluster", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-mgr-cluster-rules", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-mgr-system", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-mgr-system-rules", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-global", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-global-rules", nil) h.k8shelper.Clientset.RbacV1beta1().Roles(systemNamespace).Delete("rook-ceph-system", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rbd-csi-nodeplugin", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rbd-csi-nodeplugin", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rbd-csi-nodeplugin-rules", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rbd-csi-provisioner-role", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rbd-external-provisioner-runner", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rbd-external-provisioner-runner-rules", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("cephfs-csi-nodeplugin", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("cephfs-csi-nodeplugin", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("cephfs-csi-nodeplugin-rules", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("cephfs-csi-provisioner-role", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("cephfs-external-provisioner-runner", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("cephfs-external-provisioner-runner-rules", nil) h.k8shelper.Clientset.CoreV1().ConfigMaps(systemNamespace).Delete("csi-rbd-config", nil) h.k8shelper.Clientset.CoreV1().ConfigMaps(systemNamespace).Delete("csi-cephfs-config", nil) diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 959a0e35bf1c..556b53bd7312 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -296,6 +296,20 @@ metadata: labels: operator: rook storage-backend: ceph +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-cluster-mgmt: "true" +rules: [] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-cluster-mgmt-rules + labels: + operator: rook + storage-backend: ceph + rbac.ceph.rook.io/aggregate-to-rook-ceph-cluster-mgmt: "true" rules: - apiGroups: - "" @@ -334,6 +348,20 @@ metadata: labels: operator: rook storage-backend: ceph +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" +rules: [] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-global-rules + labels: + operator: rook + storage-backend: ceph + rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" rules: - apiGroups: - "" @@ -396,6 +424,18 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-mgr-cluster +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-mgr-cluster-rules + labels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true" rules: - apiGroups: - "" @@ -416,6 +456,20 @@ metadata: labels: operator: rook storage-backend: ceph +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-agent-mount: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-agent-mount-rules + labels: + operator: rook + storage-backend: ceph + rbac.ceph.rook.io/aggregate-to-rook-ceph-agent-mount: "true" rules: - apiGroups: - "" @@ -430,6 +484,19 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-mgr-system namespace: ` + namespace + ` +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-system: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-mgr-system-rules + namespace: ` + namespace + ` + labels: + rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-system: "true" rules: - apiGroups: - "" @@ -526,6 +593,18 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-csi-nodeplugin +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rbd-csi-nodeplugin: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin-rules + labels: + rbac.ceph.rook.io/aggregate-to-rbd-csi-nodeplugin: "true" rules: - apiGroups: [""] resources: ["nodes"] @@ -568,6 +647,18 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-external-provisioner-runner +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-external-provisioner-runner-rules + labels: + rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner: "true" rules: - apiGroups: [""] resources: ["secrets"] @@ -633,6 +724,18 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cephfs-csi-nodeplugin +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-cephfs-csi-nodeplugin: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-nodeplugin-rules + labels: + rbac.ceph.rook.io/aggregate-to-cephfs-csi-nodeplugin: "true" rules: - apiGroups: [""] resources: ["nodes"] @@ -674,6 +777,18 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cephfs-external-provisioner-runner +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-external-provisioner-runner-rules + labels: + rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner: "true" rules: - apiGroups: [""] resources: ["secrets"]