proper management for metrics updates + working e2e

Signed-off-by: Elieser Pereira <[email protected]>

Author: elieserr

operator/controllers/http/httpscaledobject_controller.go

@@ -19,6 +19,7 @@ package http
 import (
 	"context"
 	"fmt"
+	"sync"
 	"time"
 
 	kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
@@ -37,6 +38,15 @@ import (
 	"github.com/kedacore/http-add-on/operator/metrics"
 )
 
+type httpScaledObjectMetricsData struct {
+	namespace string
+}
+
+var (
+	httpScaledObjectPromMetricsMap  map[string]httpScaledObjectMetricsData
+	httpScaledObjectPromMetricsLock *sync.Mutex
+)
+
 // HTTPScaledObjectReconciler reconciles a HTTPScaledObject object
 //
 //revive:disable-next-line:exported
@@ -49,6 +59,11 @@ type HTTPScaledObjectReconciler struct {
 	BaseConfig           config.Base
 }
 
+func init() {
+	httpScaledObjectPromMetricsMap = make(map[string]httpScaledObjectMetricsData)
+	httpScaledObjectPromMetricsLock = &sync.Mutex{}
+}
+
 // +kubebuilder:rbac:groups=http.keda.sh,resources=httpscaledobjects,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=http.keda.sh,resources=httpscaledobjects/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=http.keda.sh,resources=httpscaledobjects/finalizers,verbs=update
@@ -79,6 +94,7 @@ func (r *HTTPScaledObjectReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	}
 
 	if httpso.GetDeletionTimestamp() != nil {
+		r.updatePromMetricsOnDelete(ctx, httpso)
 		return ctrl.Result{}, finalizeScaledObject(ctx, logger, r.Client, httpso)
 	}
 
@@ -99,7 +115,6 @@ func (r *HTTPScaledObjectReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		"DeploymentName",
 		httpso.Name,
 	)
-	r.updatePromMetrics(ctx, httpso, req.Namespace)
 
 	// check if ScalingMetric is correct
 	if httpso.Spec.ScalingMetric != nil &&
@@ -141,6 +156,7 @@ func (r *HTTPScaledObjectReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	)
 
 	// success reconciling
+	r.updatePromMetrics(ctx, httpso)
 	logger.Info("Reconcile success")
 	return ctrl.Result{}, nil
 }
@@ -165,14 +181,34 @@ func (r *HTTPScaledObjectReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Complete(r)
 }
 
-func (r *HTTPScaledObjectReconciler) updatePromMetrics(ctx context.Context, scaledObject *httpv1alpha1.HTTPScaledObject, namespacedName string) {
+func (r *HTTPScaledObjectReconciler) updatePromMetrics(ctx context.Context, scaledObject *httpv1alpha1.HTTPScaledObject) {
+	httpScaledObjectPromMetricsLock.Lock()
+	defer httpScaledObjectPromMetricsLock.Unlock()
+
+	namespacedName := scaledObject.Name + scaledObject.Namespace
+	metricsData, ok := httpScaledObjectPromMetricsMap[namespacedName]
+	if ok {
+		metrics.RecordDeleteHTTPScaledObjectCount(scaledObject.Namespace)
+	}
+	metricsData.namespace = scaledObject.Namespace
+
 	logger := log.FromContext(ctx, "updatePromMetrics", namespacedName)
 	logger.Info("updatePromMetrics")
-	metrics.RecordHTTPScaledObjectCount(namespacedName)
+	metrics.RecordHTTPScaledObjectCount(scaledObject.Namespace)
+
+	httpScaledObjectPromMetricsMap[namespacedName] = metricsData
 }
 
-func (r *HTTPScaledObjectReconciler) updatePromMetricsOnDelete(ctx context.Context, scaledObject *httpv1alpha1.HTTPScaledObject, namespacedName string) {
+func (r *HTTPScaledObjectReconciler) updatePromMetricsOnDelete(ctx context.Context, scaledObject *httpv1alpha1.HTTPScaledObject) {
+	httpScaledObjectPromMetricsLock.Lock()
+	defer httpScaledObjectPromMetricsLock.Unlock()
+
+	namespacedName := scaledObject.Name + scaledObject.Namespace
 	logger := log.FromContext(ctx, "updatePromMetricsOnDelete", namespacedName)
 	logger.Info("updatePromMetricsOnDelete")
-	metrics.RecordDeleteHTTPScaledObjectCount(namespacedName)
+
+	if _, ok := httpScaledObjectPromMetricsMap[namespacedName]; ok {
+		metrics.RecordDeleteHTTPScaledObjectCount(scaledObject.Namespace)
+	}
+	delete(httpScaledObjectPromMetricsMap, namespacedName)
 }

tests/checks/operator_prometheus_metrics/operator_prometheus_metrics_test.go

@@ -165,15 +165,20 @@ func TestMetricGeneration(t *testing.T) {
 	// Send a test request to the interceptor
 	sendLoad(t, kc, data)
 
+	httpScaledObjecData, httpScaledObjecDataTemplates := getTemplateHTTPScaledObjecData()
+	KubectlApplyMultipleWithTemplate(t, httpScaledObjecData, httpScaledObjecDataTemplates)
+
 	// Fetch metrics and validate them
 	family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL))
 	val, ok := family["keda_http_scaled_object_total"]
 	assert.True(t, ok, "keda_http_scaled_object_total is available")
 
-	requestCount := getMetricsValue(val)
-	assert.GreaterOrEqual(t, requestCount, float64(1))
+	httpSacaledObjectCount := getMetricsValue(val)
+	assert.GreaterOrEqual(t, httpSacaledObjectCount, float64(1))
+	assert.Equal(t, httpSacaledObjectCount, float64(2))
 
 	// cleanup
+	KubectlDeleteMultipleWithTemplate(t, httpScaledObjecData, httpScaledObjecDataTemplates)
 	DeleteKubernetesResources(t, testNamespace, data, templates)
 }
 
@@ -182,8 +187,8 @@ func sendLoad(t *testing.T, kc *kubernetes.Clientset, data templateData) {
 
 	KubectlApplyWithTemplate(t, data, "loadJobTemplate", loadJobTemplate)
 
-	assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 6, 10),
-		"replica count should be %d after 1 minutes", maxReplicaCount)
+	assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 12, 10),
+		"replica count should be %d after 2 minutes", maxReplicaCount)
 }
 
 func fetchAndParsePrometheusMetrics(t *testing.T, cmd string) map[string]*prommodel.MetricFamily {
@@ -233,3 +238,18 @@ func getTemplateData() (templateData, []Template) {
 			{Name: "httpScaledObjectTemplate", Config: httpScaledObjectTemplate},
 		}
 }
+
+func getTemplateHTTPScaledObjecData() (templateData, []Template) {
+	return templateData{
+			TestNamespace:        testNamespace,
+			DeploymentName:       "OtherDeployment",
+			ServiceName:          serviceName,
+			ClientName:           clientName,
+			HTTPScaledObjectName: "OtherHttpScaledObjectName",
+			Host:                 host,
+			MinReplicas:          minReplicaCount,
+			MaxReplicas:          maxReplicaCount,
+		}, []Template{
+			{Name: "OtherHttpScaledObjectName", Config: httpScaledObjectTemplate},
+		}
+}