From 9e988464d3e8fd425c11726b130bcc45ebf85b4c Mon Sep 17 00:00:00 2001 From: SequeI Date: Tue, 21 Oct 2025 12:03:47 +0100 Subject: [PATCH] feat: add validating webhook for run-levels Signed-off-by: SequeI --- .github/actions/kind-cluster/action.yml | 7 + .../rhtas-operator-bundle-pull-request.yaml | 2 +- Makefile | 23 +- api/v1alpha1/securesign_types.go | 1 + cmd/main.go | 12 + config/default/kustomization.yaml | 4 +- config/manager/manager.yaml | 8 + .../overlays/kubernetes/cert_resources.yaml | 21 ++ config/overlays/kubernetes/kustomization.yaml | 14 ++ config/overlays/kubernetes/webhook_patch.yaml | 6 + .../inject_ca_bundle_annotation_patch.yaml | 6 + config/overlays/openshift/kustomization.yaml | 16 ++ .../serving_cert_annotation_patch.yaml | 9 + config/webhook/kustomization.yaml | 3 + config/webhook/service.yaml | 15 ++ config/webhook/webhook.yaml | 26 ++ go.mod | 1 + internal/webhook/securesign_validator.go | 64 +++++ internal/webhook/test/webhook_test.go | 104 ++++++++ internal/webhook/webhooks.go | 20 ++ test/e2e/custom_install/suite_test.go | 222 +++++++++++++++++- 21 files changed, 568 insertions(+), 16 deletions(-) create mode 100644 config/overlays/kubernetes/cert_resources.yaml create mode 100644 config/overlays/kubernetes/kustomization.yaml create mode 100644 config/overlays/kubernetes/webhook_patch.yaml create mode 100644 config/overlays/openshift/inject_ca_bundle_annotation_patch.yaml create mode 100644 config/overlays/openshift/kustomization.yaml create mode 100644 config/overlays/openshift/serving_cert_annotation_patch.yaml create mode 100644 config/webhook/kustomization.yaml create mode 100644 config/webhook/service.yaml create mode 100644 config/webhook/webhook.yaml create mode 100644 internal/webhook/securesign_validator.go create mode 100644 internal/webhook/test/webhook_test.go create mode 100644 internal/webhook/webhooks.go diff --git a/.github/actions/kind-cluster/action.yml b/.github/actions/kind-cluster/action.yml index 9c098e10b..58db7c8e8 100644 --- a/.github/actions/kind-cluster/action.yml +++ b/.github/actions/kind-cluster/action.yml @@ -121,3 +121,10 @@ runs: shell: bash run: | kustomize build --enable-helm ./ci/nfs/overlay/ | kubectl apply -f - + + - name: Install Cert-Manager + shell: bash + run: | + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml + kubectl wait --for=condition=available deployment/cert-manager-webhook -n cert-manager --timeout=5m + kubectl wait --for=condition=available deployment/cert-manager -n cert-manager --timeout=5m diff --git a/.tekton/rhtas-operator-bundle-pull-request.yaml b/.tekton/rhtas-operator-bundle-pull-request.yaml index 01d9e9251..cc7bbb5c4 100644 --- a/.tekton/rhtas-operator-bundle-pull-request.yaml +++ b/.tekton/rhtas-operator-bundle-pull-request.yaml @@ -40,7 +40,7 @@ spec: - name: image-expires-after value: 5d - name: manager-pipelinerun-selector - value: appstudio.openshift.io/application=operator,appstudio.openshift.io/component=rhtas-operator,pipelinesascode.tekton.dev/sha={{revision}},pipelinesascode.tekton.dev/event-type in (pull_request,incoming) + value: appstudio.openshift.io/application=operator,appstudio.openshift.io/component=rhtas-operator,pipelinesascode.tekton.dev/sha={{revision}},pipelinesascode.tekton.dev/event-type in (pull_request,incoming,retest-all-comment) - name: manager-registry-url value: registry.redhat.io/rhtas/rhtas-rhel9-operator pipelineRef: diff --git a/Makefile b/Makefile index a74fe6b23..f6b2a4893 100644 --- a/Makefile +++ b/Makefile @@ -83,8 +83,20 @@ CONTAINER_TOOL ?= docker SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec +OPENSHIFT ?= + +KUSTOMIZE_OVERLAY := config/overlays/kubernetes CONFIG_DEFAULT=config/default +ifeq ($(OPENSHIFT), true) + KUSTOMIZE_OVERLAY := config/overlays/openshift + $(info Platform explicitly configured via flag/env: openshift=$(OPENSHIFT)) +else ifneq ($(OPENSHIFT),) + $(info Platform explicitly configured via flag/env: openshift=$(OPENSHIFT)) +else + $(info Platform auto-detected: openshift=false (Defaulting to Kubernetes overlay)) +endif + .PHONY: all all: build @@ -108,8 +120,8 @@ help: ## Display this help. ##@ Development .PHONY: manifests -manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases +manifests: controller-gen ## Generate ClusterRole and CustomResourceDefinition objects. + $(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./..." output:crd:artifacts:config=config/crd/bases .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. @@ -201,8 +213,7 @@ endif build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. mkdir -p dist cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build ${CONFIG_DEFAULT} > dist/install.yaml - + $(KUSTOMIZE) build ${KUSTOMIZE_OVERLAY} > dist/install.yaml ##@ Deployment ifndef ignore-not-found @@ -220,12 +231,12 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build ${CONFIG_DEFAULT} | $(KUBECTL) apply --server-side -f - + $(KUSTOMIZE) build ${KUSTOMIZE_OVERLAY} | $(KUBECTL) apply --server-side -f - .PHONY: undeploy undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build ${CONFIG_DEFAULT} | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - + $(KUSTOMIZE) build ${KUSTOMIZE_OVERLAY} | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - ##@ Dependencies diff --git a/api/v1alpha1/securesign_types.go b/api/v1alpha1/securesign_types.go index f2d115a4b..369f6c848 100644 --- a/api/v1alpha1/securesign_types.go +++ b/api/v1alpha1/securesign_types.go @@ -73,6 +73,7 @@ type SecuresignTSAStatus struct { //+kubebuilder:printcolumn:name="Rekor URL",type=string,JSONPath=`.status.rekor.url`,description="The rekor url" //+kubebuilder:printcolumn:name="Fulcio URL",type=string,JSONPath=`.status.fulcio.url`,description="The fulcio url" //+kubebuilder:printcolumn:name="Tuf URL",type=string,JSONPath=`.status.tuf.url`,description="The tuf url" +//+kubebuilder:webhook:path=/validate,mutating=false,failurePolicy=fail,groups=rhtas.redhat.com,resources=securesigns,verbs=create;update,versions=v1alpha1,name=validation.securesigns.rhtas.redhat.com,sideEffects=None,admissionReviewVersions=v1 // Securesign is the Schema for the securesigns API type Securesign struct { diff --git a/cmd/main.go b/cmd/main.go index 3db8e3df4..942b469db 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -61,6 +61,7 @@ import ( "github.com/securesign/operator/internal/controller/trillian" "github.com/securesign/operator/internal/controller/tsa" "github.com/securesign/operator/internal/controller/tuf" + rhtas_webhook "github.com/securesign/operator/internal/webhook" //+kubebuilder:scaffold:imports ) @@ -207,6 +208,17 @@ func main() { os.Exit(1) } + if err := ctrl.NewWebhookManagedBy(mgr). + For(&rhtasv1alpha1.Securesign{}). + WithValidator(&rhtas_webhook.SecureSignValidator{ + Client: mgr.GetClient(), + }). + WithValidatorCustomPath("/validate"). + Complete(); err != nil { + setupLog.Error(err, "unable to create SecureSign validating webhook") + os.Exit(1) + } + setupController("securesign", securesign.NewReconciler, mgr) setupController("fulcio", fulcio.NewReconciler, mgr) setupController("trillian", trillian.NewReconciler, mgr) diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 9bec3fc58..5f5ec7e2f 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -15,9 +15,6 @@ namePrefix: rhtas- #commonLabels: # someName: someValue -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in -# crd/kustomization.yaml -#- ../webhook # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. @@ -26,6 +23,7 @@ resources: - ../rbac - ../manager - ../prometheus +- ../webhook patches: - path: manager_metrics_patch.yaml diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index a4d146026..fd6c6de64 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -67,6 +67,10 @@ spec: - --leader-elect image: controller:latest name: manager + volumeMounts: + - name: webhook-cert + mountPath: /tmp/k8s-webhook-server/serving-certs + readOnly: true securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true @@ -97,3 +101,7 @@ spec: memory: 64Mi serviceAccountName: operator-controller-manager terminationGracePeriodSeconds: 10 + volumes: + - name: webhook-cert + secret: + secretName: webhook-server-tls diff --git a/config/overlays/kubernetes/cert_resources.yaml b/config/overlays/kubernetes/cert_resources.yaml new file mode 100644 index 000000000..460e788c2 --- /dev/null +++ b/config/overlays/kubernetes/cert_resources.yaml @@ -0,0 +1,21 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: openshift-rhtas-operator +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: webhook-serving-cert + namespace: openshift-rhtas-operator +spec: + secretName: webhook-server-tls + issuerRef: + name: selfsigned-issuer + kind: Issuer + dnsNames: + - rhtas-controller-manager-webhook-service.openshift-rhtas-operator.svc + - rhtas-controller-manager-webhook-service.openshift-rhtas-operator.svc.cluster.local diff --git a/config/overlays/kubernetes/kustomization.yaml b/config/overlays/kubernetes/kustomization.yaml new file mode 100644 index 000000000..995cf59d4 --- /dev/null +++ b/config/overlays/kubernetes/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: openshift-rhtas-operator + +resources: +- ../../default +- cert_resources.yaml + +patches: +- path: webhook_patch.yaml + target: + kind: ValidatingWebhookConfiguration + name: validation.securesigns.rhtas.redhat.com diff --git a/config/overlays/kubernetes/webhook_patch.yaml b/config/overlays/kubernetes/webhook_patch.yaml new file mode 100644 index 000000000..4ec40ee69 --- /dev/null +++ b/config/overlays/kubernetes/webhook_patch.yaml @@ -0,0 +1,6 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validation.securesigns.rhtas.redhat.com + annotations: + cert-manager.io/inject-ca-from: openshift-rhtas-operator/webhook-serving-cert diff --git a/config/overlays/openshift/inject_ca_bundle_annotation_patch.yaml b/config/overlays/openshift/inject_ca_bundle_annotation_patch.yaml new file mode 100644 index 000000000..2fb116bfc --- /dev/null +++ b/config/overlays/openshift/inject_ca_bundle_annotation_patch.yaml @@ -0,0 +1,6 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validation.securesigns.rhtas.redhat.com + annotations: + service.beta.openshift.io/inject-cabundle: "true" diff --git a/config/overlays/openshift/kustomization.yaml b/config/overlays/openshift/kustomization.yaml new file mode 100644 index 000000000..3fbd77074 --- /dev/null +++ b/config/overlays/openshift/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../default + +patches: + - path: serving_cert_annotation_patch.yaml + target: + kind: Service + name: controller-manager-webhook-service + + - path: inject_ca_bundle_annotation_patch.yaml + target: + kind: ValidatingWebhookConfiguration + name: validation.securesigns.rhtas.redhat.com diff --git a/config/overlays/openshift/serving_cert_annotation_patch.yaml b/config/overlays/openshift/serving_cert_annotation_patch.yaml new file mode 100644 index 000000000..58437d6b1 --- /dev/null +++ b/config/overlays/openshift/serving_cert_annotation_patch.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: controller-manager-webhook-service + namespace: openshift-rhtas-operator + labels: + control-plane: controller-manager + annotations: + service.beta.openshift.io/serving-cert-secret-name: webhook-server-tls diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 000000000..d226d4dca --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,3 @@ +resources: + - service.yaml + - webhook.yaml diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 000000000..520ef2061 --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: controller-manager-webhook-service + namespace: openshift-rhtas-operator + labels: + control-plane: operator-controller-manager +spec: + ports: + - name: https-webhook + port: 443 + targetPort: 9443 + protocol: TCP + selector: + control-plane: operator-controller-manager diff --git a/config/webhook/webhook.yaml b/config/webhook/webhook.yaml new file mode 100644 index 000000000..4bb04b1bd --- /dev/null +++ b/config/webhook/webhook.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validation.securesigns.rhtas.redhat.com +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: controller-manager-webhook-service + namespace: openshift-rhtas-operator + path: /validate + failurePolicy: Fail + name: validation.securesigns.rhtas.redhat.com + rules: + - apiGroups: + - rhtas.redhat.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - securesigns + sideEffects: None diff --git a/go.mod b/go.mod index f035ac1f2..513fe6870 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/operator-framework/api v0.35.0 github.com/operator-framework/operator-lib v0.19.0 github.com/robfig/cron/v3 v3.0.1 + github.com/stretchr/testify v1.11.1 golang.org/x/net v0.46.0 google.golang.org/protobuf v1.36.10 k8s.io/api v0.34.1 diff --git a/internal/webhook/securesign_validator.go b/internal/webhook/securesign_validator.go new file mode 100644 index 000000000..2cb8d8595 --- /dev/null +++ b/internal/webhook/securesign_validator.go @@ -0,0 +1,64 @@ +package webhooks + +import ( + "context" + "fmt" + + rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + logf "sigs.k8s.io/controller-runtime/pkg/log" + admission "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +func (v *SecureSignValidator) validateNamespacePolicy(ctx context.Context, operandCR *rhtasv1alpha1.Securesign) (admission.Warnings, error) { + reqLog := logf.FromContext(ctx) + targetNamespace := operandCR.GetNamespace() + + if targetNamespace == "default" { + reqLog.Info("Validation failed: Deployment blocked in 'default' namespace.") + return nil, fmt.Errorf("installation into the 'default' namespace is prohibited by RHTAS policy") + } + + ns := &corev1.Namespace{} + + if err := v.Client.Get(ctx, types.NamespacedName{Name: targetNamespace}, ns); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + reqLog.Error(err, "Failed to retrieve target namespace object for validation.") + return nil, fmt.Errorf("failed to retrieve target namespace %s: %w", targetNamespace, err) + } + + runLevel, found := ns.Labels["openshift.io/run-level"] + if found && reservedRunLevels[runLevel] { + reqLog.Info("Validation failed: Deployment blocked in reserved namespace.", + "namespace", targetNamespace, "run-level", runLevel) + return nil, fmt.Errorf("installation into reserved OpenShift namespace '%s' (run-level %s) is prohibited by RHTAS policy", targetNamespace, runLevel) + } + + return nil, nil +} + +func (v *SecureSignValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + operandCR, ok := obj.(*rhtasv1alpha1.Securesign) + if !ok { + return nil, fmt.Errorf("expected SecureSign CR but got %T", obj) + } + return v.validateNamespacePolicy(ctx, operandCR) +} + +func (v *SecureSignValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + operandCR, ok := newObj.(*rhtasv1alpha1.Securesign) + if !ok { + return nil, fmt.Errorf("expected SecureSign CR but got %T", newObj) + } + return v.validateNamespacePolicy(ctx, operandCR) +} + +func (v *SecureSignValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + // Allow all delete operations + return nil, nil +} diff --git a/internal/webhook/test/webhook_test.go b/internal/webhook/test/webhook_test.go new file mode 100644 index 000000000..543d6448a --- /dev/null +++ b/internal/webhook/test/webhook_test.go @@ -0,0 +1,104 @@ +package webhook_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/securesign/operator/api/v1alpha1" + webhook "github.com/securesign/operator/internal/webhook" +) + +func GenerateSecuresignObj(namespace string, labels map[string]string) *v1alpha1.Securesign { + return &v1alpha1.Securesign{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rhtas.redhat.com/v1alpha1", + Kind: "Securesign", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: namespace, + Labels: labels, + }, + } +} + +func TestSecureSignValidator(t *testing.T) { + mockNsReserved := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-system", + Labels: map[string]string{ + "openshift.io/run-level": "0", + }, + }, + } + mockNsAllowed := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-valid-ns", + }, + } + + c := fake.NewClientBuilder().WithObjects(mockNsReserved, mockNsAllowed).Build() + + validator := webhook.SecureSignValidator{ + Client: c, + } + + tests := []struct { + name string + obj runtime.Object + expectErr bool + }{ + { + name: "Case 1: Allowed Dynamic Namespace", + obj: GenerateSecuresignObj("test-valid-ns", nil), + expectErr: false, + }, + { + name: "Case 2: Denied Default Namespace", + obj: GenerateSecuresignObj("default", nil), + expectErr: true, + }, + { + name: "Case 3: Denied Reserved Openshift Namespace", + obj: GenerateSecuresignObj("kube-system", nil), + expectErr: true, + }, + { + name: "Case 4: Wrong Resource Type (Denial)", + obj: &unstructured.Unstructured{}, + expectErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + + _, createErr := validator.ValidateCreate(ctx, tc.obj) + if tc.expectErr { + require.Error(t, createErr, "ValidateCreate expected an error but got nil.") + } else { + require.NoError(t, createErr, "ValidateCreate returned an unexpected error.") + } + + _, updateErr := validator.ValidateUpdate(ctx, tc.obj, tc.obj) + if tc.expectErr { + require.Error(t, updateErr, "ValidateUpdate expected an error but got nil.") + } else { + require.NoError(t, updateErr, "ValidateUpdate returned an unexpected error.") + } + + _, deleteErr := validator.ValidateDelete(ctx, tc.obj) + require.NoError(t, deleteErr, "ValidateDelete unexpectedly returned an error.") + }) + } +} diff --git a/internal/webhook/webhooks.go b/internal/webhook/webhooks.go new file mode 100644 index 000000000..b5e531554 --- /dev/null +++ b/internal/webhook/webhooks.go @@ -0,0 +1,20 @@ +package webhooks + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + admission "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// SecureSignValidator checks for namespace security policy compliance. +type SecureSignValidator struct { + Client client.Client +} + +var _ admission.CustomValidator = &SecureSignValidator{} + +// Reserved OpenShift run-level labels to block +var reservedRunLevels = map[string]bool{ + "0": true, // Critical infrastructure + "1": true, // Infrastructure + "9": true, // General platform services +} diff --git a/test/e2e/custom_install/suite_test.go b/test/e2e/custom_install/suite_test.go index 9eb67e8ee..ae871183c 100644 --- a/test/e2e/custom_install/suite_test.go +++ b/test/e2e/custom_install/suite_test.go @@ -5,7 +5,11 @@ package custom_install import ( "context" "embed" + "errors" + "fmt" + "io" "os" + "strings" "testing" "time" @@ -13,17 +17,23 @@ import ( . "github.com/onsi/gomega" "github.com/onsi/gomega/format" "github.com/securesign/operator/test/e2e/support" + + admissionv1 "k8s.io/api/admissionregistration/v1" v1 "k8s.io/api/core/v1" v12 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + yamlutil "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/utils/ptr" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + runtimeCli "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/yaml" ) //go:embed testdata/* @@ -52,18 +62,34 @@ func uninstallOperator(ctx context.Context, cli runtimeCli.Client, namespace str _ = cli.Delete(ctx, pod) Eventually(func(ctx context.Context) error { return cli.Get(ctx, runtimeCli.ObjectKeyFromObject(pod), &v1.Pod{}) - }).WithContext(ctx).Should(And(HaveOccurred(), WithTransform(errors.IsNotFound, BeTrue()))) + }).WithContext(ctx).Should(And(HaveOccurred(), WithTransform(apierrors.IsNotFound, BeTrue()))) } func installOperator(ctx context.Context, cli runtimeCli.Client, ns string, opts ...optManagerPod) { for _, o := range rbac(ns) { c := o.DeepCopyObject().(runtimeCli.Object) - if e := cli.Get(ctx, runtimeCli.ObjectKeyFromObject(o), c); !errors.IsNotFound(e) { + if e := cli.Get(ctx, runtimeCli.ObjectKeyFromObject(o), c); !apierrors.IsNotFound(e) { + Expect(cli.Delete(ctx, o)).To(Succeed()) + } + Expect(cli.Create(ctx, o)).To(Succeed()) + } + + for _, o := range webhookInfra(ns) { + c := o.DeepCopyObject().(runtimeCli.Object) + if e := cli.Get(ctx, runtimeCli.ObjectKeyFromObject(o), c); !apierrors.IsNotFound(e) { Expect(cli.Delete(ctx, o)).To(Succeed()) } Expect(cli.Create(ctx, o)).To(Succeed()) } - Expect(cli.Create(ctx, managerPod(ns, opts...))).To(Succeed()) + + pod := managerPod(ns, opts...) + Expect(cli.Create(ctx, pod)).To(Succeed()) + + Expect(WaitForPodReadiness(ctx, cli, pod)).To(Succeed(), "Timed out waiting for manager Pod readiness") + + const webhookConfigName = "validation.securesigns.rhtas.redhat.com" + Expect(WaitForWebhookCaInjection(ctx, cli, webhookConfigName)).To(Succeed(), + "Timed out waiting for ValidatingWebhookConfiguration to receive CA bundle injection.") } type optManagerPod func(pod *v1.Pod) @@ -81,6 +107,9 @@ func managerPod(ns string, opts ...optManagerPod) *v1.Pod { ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: managerPodName, + Labels: map[string]string{ + "control-plane": "operator-controller-manager", + }, }, Spec: v1.PodSpec{ SecurityContext: &v1.PodSecurityContext{ @@ -94,12 +123,25 @@ func managerPod(ns string, opts ...optManagerPod) *v1.Pod { Name: "manager", Image: image, Command: []string{"/manager"}, + Ports: []v1.ContainerPort{ + { + ContainerPort: 9443, + Name: "webhook-port", + }, + }, Env: []v1.EnvVar{ { Name: "OPENSHIFT", Value: support.EnvOrDefault("OPENSHIFT", "false"), }, }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "webhook-cert", + ReadOnly: true, + MountPath: "/tmp/k8s-webhook-server/serving-certs", + }, + }, LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ @@ -123,6 +165,16 @@ func managerPod(ns string, opts ...optManagerPod) *v1.Pod { }, }, ServiceAccountName: "operator-controller-manager", + Volumes: []v1.Volume{ + { + Name: "webhook-cert", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: "webhook-server-tls", + }, + }, + }, + }, }, } @@ -143,7 +195,7 @@ func rbac(ns string) []runtimeCli.Object { Fail(err.Error()) } u := &unstructured.Unstructured{Object: map[string]interface{}{}} - if err := yaml.Unmarshal(bytes, &u); err != nil { + if err := yamlutil.Unmarshal(bytes, &u.Object); err != nil { Fail(err.Error()) } u.SetNamespace(ns) @@ -166,3 +218,161 @@ func rbac(ns string) []runtimeCli.Object { return objects } + +const ( + CertResourcesPath = "../../../config/overlays/kubernetes/cert_resources.yaml" + WebhookServicePath = "../../../config/webhook/service.yaml" + WebhookConfigPath = "../../../config/webhook/webhook.yaml" + CertManagerPatchPath = "../../../config/overlays/kubernetes/webhook_patch.yaml" +) + +func applyCertManagerAnnotationPatch(u *unstructured.Unstructured, ns string) { + patchBytes, err := os.ReadFile(CertManagerPatchPath) + if err != nil { + Fail(fmt.Errorf("failed to read CertManager patch file: %w", err).Error()) + } + + patch := &admissionv1.ValidatingWebhookConfiguration{} + + if err := yamlutil.Unmarshal(patchBytes, patch); err != nil { + Fail(fmt.Errorf("failed to unmarshal patch YAML: %w", err).Error()) + } + + baseAnnotations := u.GetAnnotations() + if baseAnnotations == nil { + baseAnnotations = make(map[string]string) + } + + const injectionAnnotationKey = "cert-manager.io/inject-ca-from" + originalAnnotationValue := patch.GetAnnotations()[injectionAnnotationKey] + + parts := strings.Split(originalAnnotationValue, "/") + certName := parts[len(parts)-1] + + newAnnotationValue := fmt.Sprintf("%s/%s", ns, certName) + + baseAnnotations[injectionAnnotationKey] = newAnnotationValue + + u.SetAnnotations(baseAnnotations) + + GinkgoWriter.Printf("Patched Webhook Config %s with Cert-Manager annotation.\n", u.GetName()) +} + +func webhookInfra(ns string) []runtimeCli.Object { + files := []string{ + CertResourcesPath, // Cert-Manager Issuer & Certificate + WebhookServicePath, // The namespaced Service + WebhookConfigPath, // The cluster-scoped ValidatingWebhookConfiguration + } + var objects = make([]runtimeCli.Object, 0) + + for _, f := range files { + bytes, err := os.ReadFile(f) + if err != nil { + Fail(fmt.Errorf("failed to read file %s: %w", f, err).Error()) + } + + decoder := yamlutil.NewYAMLOrJSONDecoder(strings.NewReader(string(bytes)), 4096) + + for { + u := &unstructured.Unstructured{Object: make(map[string]interface{})} + + if err := decoder.Decode(&u.Object); err != nil { + if errors.Is(err, io.EOF) { + break + } + Fail(fmt.Errorf("failed to decode YAML from %s: %w", f, err).Error()) + } + + kind := u.GetKind() + + if kind != "ValidatingWebhookConfiguration" { + u.SetNamespace(ns) + } + + if kind == "ValidatingWebhookConfiguration" { + applyCertManagerAnnotationPatch(u, ns) + webhooks, found, err := unstructured.NestedSlice(u.Object, "webhooks") + if !found || err != nil { + Fail(fmt.Errorf("webhook config structure missing 'webhooks' slice: %w", err).Error()) + } + + webhook := webhooks[0].(map[string]interface{}) + clientConfig := webhook["clientConfig"].(map[string]interface{}) + service := clientConfig["service"].(map[string]interface{}) + + service["namespace"] = ns + + webhooks[0] = webhook + err = unstructured.SetNestedSlice(u.Object, webhooks, "webhooks") + + if err != nil { + Fail(fmt.Errorf("failed to set Namespace on ValidatingWebhookConfiguration resource: %w", err).Error()) + } + + } + + if kind == "Certificate" { + const dynamicServiceName = "controller-manager-webhook-service" + + fqdn1 := fmt.Sprintf("%s.%s.svc", dynamicServiceName, ns) + fqdn2 := fmt.Sprintf("%s.%s.svc.cluster.local", dynamicServiceName, ns) + + err := unstructured.SetNestedStringSlice( + u.Object, + []string{fqdn1, fqdn2}, + "spec", + "dnsNames", + ) + if err != nil { + Fail(fmt.Errorf("failed to set dnsNames on Certificate resource: %w", err).Error()) + } + } + + objects = append(objects, u) + } + } + + return objects +} + +func WaitForPodReadiness(ctx context.Context, cli runtimeCli.Client, pod *v1.Pod) error { + return wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, false, func(ctx context.Context) (bool, error) { + p := &v1.Pod{} + key := types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace} + + if err := cli.Get(ctx, key, p); err != nil { + return false, nil + } + + for _, condition := range p.Status.Conditions { + if condition.Type == v1.PodReady && condition.Status == v1.ConditionTrue { + return true, nil + } + } + return false, nil + }) +} + +func WaitForWebhookCaInjection(ctx context.Context, cli runtimeCli.Client, webhookConfigName string) error { + const timeout = 2 * time.Minute + const pollInterval = 5 * time.Second + + key := types.NamespacedName{Name: webhookConfigName} + + return wait.PollUntilContextTimeout(ctx, pollInterval, timeout, false, func(ctx context.Context) (bool, error) { + config := &admissionv1.ValidatingWebhookConfiguration{} + + if err := cli.Get(ctx, key, config); err != nil { + return false, nil + } + + for _, webhook := range config.Webhooks { + if len(webhook.ClientConfig.CABundle) > 0 { + return true, nil + } + } + + return false, nil + }) +}