document how to use prometheus and otel metrics. change matric type to fit meaning

Signed-off-by: Elieser Pereira <[email protected]>

Author: elieserr

CHANGELOG.md

@@ -33,6 +33,7 @@ This changelog keeps track of work items that have been completed and are ready
 ### Improvements
 
 - **Interceptor**: Support HTTPScaledObject scoped timeout ([#813](https://github.com/kedacore/http-add-on/issues/813))
+- **General**: Add prometehus and otel instrumentation for the operator ([#965](https://github.com/kedacore/http-add-on/issues/965))
 
 ### Fixes
 

docs/operate.md

@@ -60,3 +60,22 @@ Optional variables
 `OTEL_EXPORTER_OTLP_TRACES_TIMEOUT` - The batcher timeout in seconds to send batch of data points (`5` by default)
 
 ### Configuring Service Failover
+
+# Configuring metrics for the KEDA HTTP Add-on Operator
+
+### Exportable metrics:
+* **operator_http_scaled_object_count** - the number of http_scaled_object  
+
+There are currently 2 supported methods for exposing metrics from the operator - via a Prometheus compatible metrics endpoint or by pushing metrics to a OTEL HTTP collector.
+
+### Configuring the Prometheus compatible metrics endpoint
+When configured, the operator can expose metrics on a Prometheus compatible endpoint.
+
+This endpoint can be enabled by setting the `OTEL_PROM_EXPORTER_ENABLED` environment variable to `true` on the operator deployment (`true` by default) and by setting `OTEL_PROM_EXPORTER_PORT` to an unused port for the endpoint to be made avaialble on (`2223` by default).
+
+### Configuring the OTEL HTTP exporter
+When configured, the ioperator can export metrics to a OTEL HTTP collector.
+
+The OTEL exporter can be enabled by setting the `OTEL_EXPORTER_OTLP_METRICS_ENABLED` environment variable to `true` on the operator deployment (`false` by default). When enabled the `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable must also be configured so the exporter knows what collector to send the metrics to (e.g. http://opentelemetry-collector.open-telemetry-system:4318).
+
+If you need to provide any headers such as authentication details in order to utilise your OTEL collector you can add them into the `OTEL_EXPORTER_OTLP_HEADERS` environment variable. The frequency at which the metrics are exported can be configured by setting `OTEL_METRIC_EXPORT_INTERVAL` to the number of seconds you require between each export interval (`30` by default).

operator/controllers/http/httpscaledobject_controller.go

@@ -170,3 +170,9 @@ func (r *HTTPScaledObjectReconciler) updatePromMetrics(ctx context.Context, scal
 	logger.Info("updatePromMetrics")
 	metrics.RecordHTTPScaledObjectCount(namespacedName)
 }
+
+func (r *HTTPScaledObjectReconciler) updatePromMetricsOnDelete(ctx context.Context, scaledObject *httpv1alpha1.HTTPScaledObject, namespacedName string) {
+	logger := log.FromContext(ctx, "updatePromMetricsOnDelete", namespacedName)
+	logger.Info("updatePromMetricsOnDelete")
+	metrics.RecordDeleteHTTPScaledObjectCount(namespacedName)
+}

operator/metrics/metricscollector.go

@@ -12,6 +12,7 @@ const meterName = "keda-http-add-on-operator"
 
 type Collector interface {
 	RecordHTTPScaledObjectCount(namespace string)
+	RecordDeleteHTTPScaledObjectCount(namespace string)
 }
 
 func NewMetricsCollectors(metricsConfig *config.Metrics) {
@@ -31,3 +32,9 @@ func RecordHTTPScaledObjectCount(namespace string) {
 		collector.RecordHTTPScaledObjectCount(namespace)
 	}
 }
+
+func RecordDeleteHTTPScaledObjectCount(namespace string) {
+	for _, collector := range collectors {
+		collector.RecordDeleteHTTPScaledObjectCount(namespace)
+	}
+}

operator/metrics/otelmetrics.go

@@ -16,7 +16,7 @@ import (
 
 type OtelMetrics struct {
 	meter                   api.Meter
-	httpScaledObjectCounter api.Int64Counter
+	httpScaledObjectCounter api.Int64UpDownCounter
 }
 
 func NewOtelMetrics(options ...metric.Option) *OtelMetrics {
@@ -43,7 +43,7 @@ func NewOtelMetrics(options ...metric.Option) *OtelMetrics {
 	provider := metric.NewMeterProvider(options...)
 	meter := provider.Meter(meterName)
 
-	httpScaledObjectCounter, err := meter.Int64Counter("operator_http_scaled_object_count", api.WithDescription("a counter of http_scaled_objects processed by the operator"))
+	httpScaledObjectCounter, err := meter.Int64UpDownCounter("operator_http_scaled_object_count", api.WithDescription("a counter of http_scaled_objects processed by the operator"))
 	if err != nil {
 		log.Fatalf("could not create new otelhttpmetric request counter: %v", err)
 	}
@@ -63,3 +63,13 @@ func (om *OtelMetrics) RecordHTTPScaledObjectCount(namespace string) {
 	)
 	om.httpScaledObjectCounter.Add(ctx, 1, opt)
 }
+
+func (om *OtelMetrics) RecordDeleteHTTPScaledObjectCount(namespace string) {
+	ctx := context.Background()
+	opt := api.WithAttributeSet(
+		attribute.NewSet(
+			attribute.Key("namespace").String(namespace),
+		),
+	)
+	om.httpScaledObjectCounter.Add(ctx, -1, opt)
+}

operator/metrics/prommetrics.go

@@ -16,7 +16,7 @@ import (
 
 type PrometheusMetrics struct {
 	meter                   api.Meter
-	httpScaledObjectCounter api.Int64Counter
+	httpScaledObjectCounter api.Int64UpDownCounter
 }
 
 func NewPrometheusMetrics(options ...prometheus.Option) *PrometheusMetrics {
@@ -43,7 +43,7 @@ func NewPrometheusMetrics(options ...prometheus.Option) *PrometheusMetrics {
 	)
 	meter := provider.Meter(meterName)
 
-	httpScaledObjectCounter, err := meter.Int64Counter("operator_http_scaled_object_count", api.WithDescription("a counter of http_scaled_objects processed by the operator"))
+	httpScaledObjectCounter, err := meter.Int64UpDownCounter("operator_http_scaled_object_count", api.WithDescription("a counter of http_scaled_objects processed by the operator"))
 	if err != nil {
 		log.Fatalf("could not create new Prometheus request counter: %v", err)
 	}
@@ -63,3 +63,13 @@ func (p *PrometheusMetrics) RecordHTTPScaledObjectCount(namespace string) {
 	)
 	p.httpScaledObjectCounter.Add(ctx, 1, opt)
 }
+
+func (p *PrometheusMetrics) RecordDeleteHTTPScaledObjectCount(namespace string) {
+	ctx := context.Background()
+	opt := api.WithAttributeSet(
+		attribute.NewSet(
+			attribute.Key("namespace").String(namespace),
+		),
+	)
+	p.httpScaledObjectCounter.Add(ctx, -1, opt)
+}

operator/metrics/prommetrics_test.go

@@ -15,10 +15,10 @@ func TestPromRequestCountMetric(t *testing.T) {
 	options := []promexporter.Option{promexporter.WithRegisterer(testRegistry)}
 	testPrometheus := NewPrometheusMetrics(options...)
 	expectedOutput := `
-	# HELP operator_http_scaled_object_count_total a counter of http_scaled_objects processed by the operator
-	# TYPE operator_http_scaled_object_count_total counter
-	operator_http_scaled_object_count_total{namespace="test-namespace",otel_scope_name="keda-http-add-on-operator",otel_scope_version=""} 2
-	operator_http_scaled_object_count_total{namespace="other-test-namespace",otel_scope_name="keda-http-add-on-operator",otel_scope_version=""} 1
+	# HELP operator_http_scaled_object_count a counter of http_scaled_objects processed by the operator
+	# TYPE operator_http_scaled_object_count gauge
+	operator_http_scaled_object_count{namespace="test-namespace",otel_scope_name="keda-http-add-on-operator",otel_scope_version=""} 0
+	operator_http_scaled_object_count{namespace="other-test-namespace",otel_scope_name="keda-http-add-on-operator",otel_scope_version=""} 1
 	# HELP otel_scope_info Instrumentation Scope metadata
 	# TYPE otel_scope_info gauge
 	otel_scope_info{otel_scope_name="keda-http-add-on-operator",otel_scope_version=""} 1
@@ -28,7 +28,7 @@ func TestPromRequestCountMetric(t *testing.T) {
 	`
 	expectedOutputReader := strings.NewReader(expectedOutput)
 	testPrometheus.RecordHTTPScaledObjectCount("test-namespace")
-	testPrometheus.RecordHTTPScaledObjectCount("test-namespace")
+	testPrometheus.RecordDeleteHTTPScaledObjectCount("test-namespace")
 	testPrometheus.RecordHTTPScaledObjectCount("other-test-namespace")
 	err := testutil.CollectAndCompare(testRegistry, expectedOutputReader)
 	assert.Nil(t, err)