register metrics collector to use existing metrics server

Signed-off-by: Elieser Pereira [email protected]

Author: elieserr

config/operator/deployment.yaml

@@ -38,7 +38,7 @@ spec:
           value: ""
         ports:
         - name: metrics
-          containerPort: 2223
+          containerPort: 8080
         - name: probes
           containerPort: 8081
         livenessProbe:

config/operator/e2e-test/otel/deployment.yaml

@@ -12,10 +12,11 @@ spec:
         - name: OTEL_PROM_EXPORTER_ENABLED
           value: "true"
         - name: OTEL_PROM_EXPORTER_PORT
-          value: "2223"
+          value: "8080"
         - name: OTEL_EXPORTER_OTLP_METRICS_ENABLED
           value: "true"
         - name: OTEL_EXPORTER_OTLP_ENDPOINT
           value: "http://opentelemetry-collector.open-telemetry-system:4318"
         - name: OTEL_METRIC_EXPORT_INTERVAL
           value: "1"
+

config/operator/metrics.service.yaml

@@ -7,5 +7,5 @@ spec:
   ports:
   - name: metrics
     protocol: TCP
-    port: 2223
+    port: 8080
     targetPort: metrics

operator/controllers/http/config/config.go

@@ -96,7 +96,7 @@ type Metrics struct {
 	// Sets whether or not to enable the Prometheus metrics exporter
 	OtelPrometheusExporterEnabled bool `envconfig:"OTEL_PROM_EXPORTER_ENABLED" default:"true"`
 	// Sets the port which the Prometheus compatible metrics endpoint should be served on
-	OtelPrometheusExporterPort int `envconfig:"OTEL_PROM_EXPORTER_PORT" default:"2223"`
+	OtelPrometheusExporterPort int `envconfig:"OTEL_PROM_EXPORTER_PORT" default:"8080"`
 	// Sets whether or not to enable the OTEL metrics exporter
 	OtelHTTPExporterEnabled bool `envconfig:"OTEL_EXPORTER_OTLP_METRICS_ENABLED" default:"false"`
 }

operator/main.go

@@ -17,14 +17,15 @@ limitations under the License.
 package main
 
 import (
-	"context"
 	"flag"
-	"fmt"
 	"os"
 
-	"github.com/go-logr/logr"
+	httpv1alpha1 "github.com/kedacore/http-add-on/operator/apis/http/v1alpha1"
+	httpcontrollers "github.com/kedacore/http-add-on/operator/controllers/http"
+	"github.com/kedacore/http-add-on/operator/controllers/http/config"
+	"github.com/kedacore/http-add-on/operator/metrics"
+	"github.com/kedacore/http-add-on/pkg/util"
 	kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -34,14 +35,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
-
-	httpv1alpha1 "github.com/kedacore/http-add-on/operator/apis/http/v1alpha1"
-	httpcontrollers "github.com/kedacore/http-add-on/operator/controllers/http"
-	"github.com/kedacore/http-add-on/operator/controllers/http/config"
-	"github.com/kedacore/http-add-on/operator/metrics"
-	kedahttp "github.com/kedacore/http-add-on/pkg/http"
-	"github.com/kedacore/http-add-on/pkg/util"
-	"golang.org/x/sync/errgroup"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -124,20 +117,6 @@ func main() {
 
 	ctx := ctrl.SetupSignalHandler()
 	ctx = util.ContextWithLogger(ctx, ctrl.Log)
-	eg, ctx := errgroup.WithContext(ctx)
-
-	if metricsCfg.OtelPrometheusExporterEnabled {
-		// start the prometheus compatible metrics server
-		// serves a prometheus compatible metrics endpoint on the configured port
-		eg.Go(func() error {
-			if err := runMetricsServer(ctx, ctrl.Log, metricsCfg); !util.IsIgnoredErr(err) {
-				setupLog.Error(err, "could not start the Prometheus metrics server")
-				return err
-			}
-
-			return nil
-		})
-	}
 	if err = (&httpcontrollers.HTTPScaledObjectReconciler{
 		Client: mgr.GetClient(),
 		Scheme: mgr.GetScheme(),
@@ -165,13 +144,3 @@ func main() {
 		os.Exit(1)
 	}
 }
-
-func runMetricsServer(
-	ctx context.Context,
-	lggr logr.Logger,
-	metricsCfg *config.Metrics,
-) error {
-	lggr.Info("starting the prometheus metrics server", "port", metricsCfg.OtelPrometheusExporterPort, "path", "/metrics")
-	addr := fmt.Sprintf("0.0.0.0:%d", metricsCfg.OtelPrometheusExporterPort)
-	return kedahttp.ServeContext(ctx, addr, promhttp.Handler(), nil)
-}

operator/metrics/metricscollector.go

@@ -2,6 +2,8 @@ package metrics
 
 import (
 	"github.com/kedacore/http-add-on/operator/controllers/http/config"
+	"go.opentelemetry.io/otel/exporters/prometheus"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
 var (
@@ -17,7 +19,8 @@ type Collector interface {
 
 func NewMetricsCollectors(metricsConfig *config.Metrics) {
 	if metricsConfig.OtelPrometheusExporterEnabled {
-		promometrics := NewPrometheusMetrics()
+		options := prometheus.WithRegisterer(ctrlmetrics.Registry)
+		promometrics := NewPrometheusMetrics(options)
 		collectors = append(collectors, promometrics)
 	}
 

operator/metrics/prommetrics.go

@@ -4,14 +4,13 @@ import (
 	"context"
 	"log"
 
+	"github.com/kedacore/http-add-on/pkg/build"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/exporters/prometheus"
 	api "go.opentelemetry.io/otel/metric"
 	"go.opentelemetry.io/otel/sdk/metric"
 	"go.opentelemetry.io/otel/sdk/resource"
 	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
-
-	"github.com/kedacore/http-add-on/pkg/build"
 )
 
 type PrometheusMetrics struct {

tests/checks/operator_otel_metrics/operator_otel_metrics_test.go

@@ -167,16 +167,16 @@ func TestMetricGeneration(t *testing.T) {
 
 	// Fetch metrics and validate them
 	family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", otelCollectorPromURL))
-	val, ok := family["operator_http_scaled_object_count_total"]
+	val, ok := family["operator_http_scaled_object_count"]
 	// If the metric is not found first time around then retry with a delay.
 	if !ok {
 		// Add a small sleep to allow metrics to be pushed from the exporter to the collector
 		time.Sleep(10 * time.Second)
 		// Fetch metrics and validate them
 		family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", otelCollectorPromURL))
-		val, ok = family["operator_http_scaled_object_count_total"]
+		val, ok = family["operator_http_scaled_object_count"]
 	}
-	assert.True(t, ok, "operator_http_scaled_object_count_total is available")
+	assert.True(t, ok, "operator_http_scaled_object_count is available")
 
 	requestCount := getMetricsValue(val)
 	assert.GreaterOrEqual(t, requestCount, float64(1))
@@ -208,13 +208,13 @@ func fetchAndParsePrometheusMetrics(t *testing.T, cmd string) map[string]*prommo
 }
 
 func getMetricsValue(val *prommodel.MetricFamily) float64 {
-	if val.GetName() == "operator_http_scaled_object_count_total" {
+	if val.GetName() == "operator_http_scaled_object_count" {
 		metrics := val.GetMetric()
 		for _, metric := range metrics {
 			labels := metric.GetLabel()
 			for _, label := range labels {
 				if *label.Name == "namespace" && *label.Value == testNamespace {
-					return metric.GetCounter().GetValue()
+					return metric.GetGauge().GetValue()
 				}
 			}
 		}

tests/checks/operator_prometheus_metrics/operator_prometheus_metrics_test.go

@@ -29,7 +29,7 @@ var (
 	host                      = testName
 	minReplicaCount           = 0
 	maxReplicaCount           = 1
-	kedaOperatorPrometheusURL = "http://keda-add-ons-http-operator-metrics.keda:2223/metrics"
+	kedaOperatorPrometheusURL = "http://keda-add-ons-http-operator-metrics.keda:8080/metrics"
 )
 
 type templateData struct {
@@ -167,13 +167,11 @@ func TestMetricGeneration(t *testing.T) {
 
 	// Fetch metrics and validate them
 	family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL))
-	val, ok := family["operator_http_scaled_object_count_total"]
-	assert.True(t, ok, "operator_http_scaled_object_count_total is available")
-	t.Log("--- ASSERT operator_http_scaled_object_count_total is available ---")
+	val, ok := family["operator_http_scaled_object_count"]
+	assert.True(t, ok, "operator_http_scaled_object_count is available")
 
 	requestCount := getMetricsValue(val)
 	assert.GreaterOrEqual(t, requestCount, float64(1))
-	t.Log("--- ASSERT metrics greater than1 ---")
 
 	// cleanup
 	DeleteKubernetesResources(t, testNamespace, data, templates)
@@ -203,13 +201,14 @@ func fetchAndParsePrometheusMetrics(t *testing.T, cmd string) map[string]*prommo
 }
 
 func getMetricsValue(val *prommodel.MetricFamily) float64 {
-	if val.GetName() == "operator_http_scaled_object_count_total" {
+	if val.GetName() == "operator_http_scaled_object_count" {
+
 		metrics := val.GetMetric()
 		for _, metric := range metrics {
 			labels := metric.GetLabel()
 			for _, label := range labels {
 				if *label.Name == "namespace" && *label.Value == testNamespace {
-					return metric.GetCounter().GetValue()
+					return metric.GetGauge().GetValue()
 				}
 			}
 		}