diff --git a/.gitignore b/.gitignore index 795b07354..96ec6df46 100644 --- a/.gitignore +++ b/.gitignore @@ -362,3 +362,4 @@ admin/Cargo.lock *.csr *.srl *.ext +*.test diff --git a/CHANGELOG.md b/CHANGELOG.md index 13098cf0e..c8780cbb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,12 @@ This changelog keeps track of work items that have been completed and are ready - **General**: TODO ([#TODO](https://github.com/kedacore/http-add-on/issues/TODO)) +## v0.12.0 + +### Improvements + +- **General**: Get rid of kube-rbac-proxy ([#1123](https://github.com/kedacore/http-add-on/pull/1369)) + ## v0.11.1 ### Improvements diff --git a/config/operator/deployment.yaml b/config/operator/deployment.yaml index d86706dd6..bcb504b50 100644 --- a/config/operator/deployment.yaml +++ b/config/operator/deployment.yaml @@ -38,7 +38,7 @@ spec: value: "" ports: - name: metrics - containerPort: 8080 + containerPort: 8443 - name: probes containerPort: 8081 livenessProbe: diff --git a/config/operator/kustomization.yaml b/config/operator/kustomization.yaml index 08e075759..910b833fc 100644 --- a/config/operator/kustomization.yaml +++ b/config/operator/kustomization.yaml @@ -2,6 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - deployment.yaml +- metrics_service.yaml +- metrics_reader_role.yaml - role.yaml - role_binding.yaml - service_account.yaml diff --git a/config/operator/metrics_reader_role.yaml b/config/operator/metrics_reader_role.yaml new file mode 100644 index 000000000..c26634869 --- /dev/null +++ b/config/operator/metrics_reader_role.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: operator-metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/config/operator/metrics_service.yaml b/config/operator/metrics_service.yaml new file mode 100644 index 000000000..2c879f799 --- /dev/null +++ b/config/operator/metrics_service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: operator-metrics +spec: + type: ClusterIP + ports: + - name: metrics + protocol: TCP + port: 8443 + targetPort: 8443 diff --git a/config/operator/role.yaml b/config/operator/role.yaml index f89c4d39f..eac4a4dac 100644 --- a/config/operator/role.yaml +++ b/config/operator/role.yaml @@ -42,6 +42,18 @@ rules: - patch - update - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role diff --git a/config/operator/role_binding.yaml b/config/operator/role_binding.yaml index e8bf02c81..d078dc02e 100644 --- a/config/operator/role_binding.yaml +++ b/config/operator/role_binding.yaml @@ -2,23 +2,39 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: operator + name: keda-http-operator-auth-delegator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: operator + name: system:auth-delegator subjects: - kind: ServiceAccount + name: keda-add-ons-http-operator + namespace: keda +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: keda-http-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole name: operator +subjects: +- kind: ServiceAccount + name: keda-add-ons-http-operator + namespace: keda --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: operator + namespace: keda roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: operator subjects: - kind: ServiceAccount - name: operator + name: keda-add-ons-http-operator + namespace: keda diff --git a/go.mod b/go.mod index d8c687612..eb800ca3e 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,16 @@ require ( sigs.k8s.io/kustomize/kustomize/v5 v5.7.1 ) +require ( + cel.dev/expr v0.24.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/google/cel-go v0.26.0 // indirect + github.com/stoewer/go-strcase v1.3.1 // indirect + k8s.io/apiserver v0.33.5 // indirect + k8s.io/component-base v0.33.5 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect +) + require ( github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index 3b139f000..c2be2bb48 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -64,6 +68,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.22.0 h1:b3FJZxpiv1vTMo2/5RDUqAHPxkT8mmMfJIrq1llbf7g= +github.com/google/cel-go v0.22.0/go.mod h1:BuznPXXfQDpXKWQ9sPW3TzlAJN5zzFe+i9tIs0yC4s8= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -172,6 +178,8 @@ github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wx github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= +github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -328,10 +336,14 @@ k8s.io/apiextensions-apiserver v0.33.5 h1:93NZh6rmrcamX/tfv/dZrTsMiQX69ufANmDcKP k8s.io/apiextensions-apiserver v0.33.5/go.mod h1:JIbyQnNlu6nQa7b1vgFi51pmlXOk8mdn0WJwUJnz/7U= k8s.io/apimachinery v0.33.5 h1:NiT64hln4TQXeYR18/ES39OrNsjGz8NguxsBgp+6QIo= k8s.io/apimachinery v0.33.5/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/apiserver v0.33.5 h1:X1Gy33r4YkRLRqTjGjofk7X1/EjSLEVSJ/A+1qjoj60= +k8s.io/apiserver v0.33.5/go.mod h1:Q+b5Btbc8x0PqOCeh/xBTesKk+cXQRN+PF2wdrTKDeg= k8s.io/client-go v0.33.5 h1:I8BdmQGxInpkMEnJvV6iG7dqzP3JRlpZZlib3OMFc3o= k8s.io/client-go v0.33.5/go.mod h1:W8PQP4MxbM4ypgagVE65mUUqK1/ByQkSALF9tzuQ6u0= k8s.io/code-generator v0.33.5 h1:KwkOvhwAaorjSwF2MQhhdhL3i8bBmAal/TWhX67kdHw= k8s.io/code-generator v0.33.5/go.mod h1:Ra+sdZquRakeTGcEnQAPw6BmlZ92IvxwQQTX/XOvOIE= +k8s.io/component-base v0.33.5 h1:4D3kxjEx1pJRy3WHAZsmX3+LCpmd4ftE+2J4v6naTnQ= +k8s.io/component-base v0.33.5/go.mod h1:Zma1YjBVuuGxIbspj1vGR3/5blzo2ARf1v0QTtog1to= k8s.io/gengo/v2 v2.0.0-20250820003526-c297c0c1eb9d h1:qUrYOinhdAUL0xxhA4gPqogPBaS9nIq2l2kTb6pmeB0= k8s.io/gengo/v2 v2.0.0-20250820003526-c297c0c1eb9d/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= @@ -342,6 +354,8 @@ k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzk k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= knative.dev/pkg v0.0.0-20250602175424-3c3a920206ea h1:ukJPq9MzFTEH/Sei5MSVnSE8+7NSCKixCDZPd6p4ohw= knative.dev/pkg v0.0.0-20250602175424-3c3a920206ea/go.mod h1:tFayQbi6t4+5HXuEGLOGvILW228Q7uaJp/FYEgbjJ3A= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= sigs.k8s.io/controller-tools v0.17.3 h1:lwFPLicpBKLgIepah+c8ikRBubFW5kOQyT88r3EwfNw= diff --git a/operator/main.go b/operator/main.go index 4844a268a..ea6478e20 100644 --- a/operator/main.go +++ b/operator/main.go @@ -29,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" "sigs.k8s.io/controller-runtime/pkg/metrics/server" httpv1alpha1 "github.com/kedacore/http-add-on/operator/apis/http/v1alpha1" @@ -58,7 +59,7 @@ func main() { var enableLeaderElection bool var probeAddr string var profilingAddr string - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8443", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ @@ -96,7 +97,9 @@ func main() { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, Metrics: server.Options{ - BindAddress: metricsAddr, + BindAddress: metricsAddr, + SecureServing: true, + FilterProvider: filters.WithAuthenticationAndAuthorization, }, PprofBindAddress: profilingAddr, HealthProbeBindAddress: probeAddr, diff --git a/tests/checks/operator_metrics/operator_metrics_test.go b/tests/checks/operator_metrics/operator_metrics_test.go new file mode 100644 index 000000000..fb4ec236c --- /dev/null +++ b/tests/checks/operator_metrics/operator_metrics_test.go @@ -0,0 +1,140 @@ +//go:build e2e +// +build e2e + +package operator_metrics_test + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + . "github.com/kedacore/http-add-on/tests/helper" +) + +const ( + testName = "operator-metrics-test" +) + +var ( + testNamespace = fmt.Sprintf("%s-ns", testName) + clientName = fmt.Sprintf("%s-client", testName) + kedaOperatorMetricsURL = "https://keda-add-ons-http-operator-metrics.keda:8443/metrics" + operatorPodSelector = "app.kubernetes.io/instance=operator" +) + +type templateData struct { + TestNamespace string + ClientName string +} + +const ( + clientTemplate = ` +apiVersion: v1 +kind: Pod +metadata: + name: {{.ClientName}} + namespace: {{.TestNamespace}} +spec: + serviceAccountName: {{.ClientName}} + containers: + - name: {{.ClientName}} + image: curlimages/curl + command: + - sh + - -c + - "exec tail -f /dev/null"` + + serviceAccountTemplate = ` +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{.ClientName}} + namespace: {{.TestNamespace}}` + + clusterRoleBindingTemplate = ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{.ClientName}}-metrics-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: keda-add-ons-http-operator-metrics-reader +subjects: +- kind: ServiceAccount + name: {{.ClientName}} + namespace: {{.TestNamespace}}` +) + +func TestOperatorMetrics(t *testing.T) { + // setup + t.Log("--- setting up ---") + // Create kubernetes resources + kc := GetKubernetesClient(t) + data, templates := getTemplateData() + CreateKubernetesResources(t, kc, testNamespace, data, templates) + + // Wait for client pod to be ready + assert.True(t, WaitForAllPodRunningInNamespace(t, kc, testNamespace, 6, 10), + "client pod should be running") + + t.Log("--- testing operator metrics endpoint ---") + + // Test 1: HTTPS endpoint should be accessible (will fail cert validation but should return metrics) + t.Log("Test 1: Verify HTTPS endpoint is available") + testHTTPSEndpoint(t) + + // Test 2: Verify metrics are returned + t.Log("Test 2: Verify metrics content") + testMetricsContent(t) + + // cleanup + DeleteKubernetesResources(t, testNamespace, data, templates) +} + +func testHTTPSEndpoint(t *testing.T) { + // Use curl with -k to skip certificate validation (self-signed cert) + cmd := fmt.Sprintf("curl -k --max-time 10 %s", kedaOperatorMetricsURL) + out, errOut, err := ExecCommandOnSpecificPod(t, clientName, testNamespace, cmd) + + // We expect this to succeed with a self-signed certificate + if err != nil { + t.Logf("HTTPS endpoint test - Output: %s, Error output: %s, Error: %v", out, errOut, err) + } + + // The endpoint should return something (even if authentication fails, it should respond) + assert.True(t, err == nil || strings.Contains(errOut, "Forbidden") || strings.Contains(out, "Forbidden"), + "HTTPS endpoint should respond (either with metrics or authentication error)") +} + +func testMetricsContent(t *testing.T) { + // Access metrics from the client pod using the service endpoint + // The client pod uses a ServiceAccount with the operator-metrics-reader ClusterRole + // This allows it to access the metrics endpoint with proper RBAC permissions + + // Get the ServiceAccount token to authenticate + cmd := fmt.Sprintf("curl -k -H \"Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" --max-time 10 %s", kedaOperatorMetricsURL) + out, errOut, err := ExecCommandOnSpecificPod(t, clientName, testNamespace, cmd) + + if err != nil { + t.Logf("Metrics content test - Output: %s, Error output: %s, Error: %v", out, errOut, err) + } + + // Verify that metrics are returned in Prometheus format + assert.NoError(t, err, "should be able to access metrics from client pod with RBAC permissions. Output: %s, Error: %s", out, errOut) + assert.True(t, strings.Contains(out, "# HELP") || strings.Contains(out, "# TYPE"), + "metrics should contain Prometheus format. Output: %s", out) +} + +func getTemplateData() (templateData, []Template) { + return templateData{ + TestNamespace: testNamespace, + ClientName: clientName, + }, []Template{ + {Name: "serviceAccountTemplate", Config: serviceAccountTemplate}, + {Name: "clusterRoleBindingTemplate", Config: clusterRoleBindingTemplate}, + {Name: "clientTemplate", Config: clientTemplate}, + } +} diff --git a/tests/utils/setup_test.go b/tests/utils/setup_test.go index 95a38608e..aa371a180 100644 --- a/tests/utils/setup_test.go +++ b/tests/utils/setup_test.go @@ -66,7 +66,13 @@ kind: Deployment metadata: name: opentelemetry-collector spec: + selector: + matchLabels: + app.kubernetes.io/name: opentelemetry-collector template: + metadata: + labels: + app.kubernetes.io/name: opentelemetry-collector spec: containers: - name: opentelemetry-collector @@ -123,16 +129,16 @@ spec: app: zipkin spec: containers: - - image: openzipkin/zipkin + - image: openzipkin/zipkin:3 name: zipkin env: - name: "JAVA_OPTS" - value: "-Xmx500M" + value: "-XX:MaxRAMPercentage=80 -XX:InitialRAMPercentage=80" resources: limits: - memory: "700M" + memory: "256M" requests: - memory: "5M" + memory: "256M" --- apiVersion: v1 kind: Service @@ -313,7 +319,7 @@ func TestDeployZipkin(t *testing.T) { _, err = ExecuteCommand(fmt.Sprintf("kubectl apply -f %s -n %s", zipkinTemplateFileName, "zipkin")) require.NoErrorf(t, err, "cannot deploy zipkin - %s", err) - assert.True(t, WaitForDeploymentReplicaReadyCount(t, KubeClient, "zipkin", "zipkin", 1, 6, 5), + assert.True(t, WaitForDeploymentReplicaReadyCount(t, KubeClient, "zipkin", "zipkin", 1, 6, 10), "replica count should be 1 after 3 minutes") }