feat(crd): Refactor prometheus metrics

Co-authored-by: Gemini AI <[email protected]>"
Signed-off-by: Denis Karpelevich <[email protected]>

Author: dkarpele

cmd/common.go

@@ -14,6 +14,7 @@ import (
 	"github.com/argoproj-labs/argocd-image-updater/internal/controller"
 	"github.com/argoproj-labs/argocd-image-updater/pkg/argocd"
 	"github.com/argoproj-labs/argocd-image-updater/pkg/common"
+	"github.com/argoproj-labs/argocd-image-updater/pkg/metrics"
 	"github.com/argoproj-labs/argocd-image-updater/pkg/webhook"
 	"github.com/argoproj-labs/argocd-image-updater/registry-scanner/pkg/registry"
 )
@@ -30,6 +31,9 @@ type WebhookConfig struct {
 
 // SetupCommon initializes common components (logging, context, etc.)
 func SetupCommon(ctx context.Context, cfg *controller.ImageUpdaterConfig, setupLogger logr.Logger, commitMessagePath, kubeConfig string) error {
+	// Initialize metrics before starting the metrics server or using any counters
+	metrics.InitMetrics()
+
 	var commitMessageTpl string
 
 	// User can specify a path to a template used for Git commit messages

cmd/common_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/argoproj-labs/argocd-image-updater/internal/controller"
 	"github.com/argoproj-labs/argocd-image-updater/pkg/common"
 	aiukube "github.com/argoproj-labs/argocd-image-updater/pkg/kube"
+	"github.com/argoproj-labs/argocd-image-updater/pkg/metrics"
 	"github.com/argoproj-labs/argocd-image-updater/registry-scanner/pkg/registry"
 )
 
@@ -64,6 +65,8 @@ var setupCommonMutex sync.Mutex
 
 // setupCommonStub mirrors SetupCommon behavior without starting the askpass server and without interactive kube client.
 func setupCommonStub(ctx context.Context, cfg *controller.ImageUpdaterConfig, setupLogger logr.Logger, commitMessagePath, kubeConfig string) error {
+	metrics.InitMetrics()
+
 	var commitMessageTpl string
 
 	// User can specify a path to a template used for Git commit messages
@@ -231,4 +234,15 @@ func TestSetupCommon(t *testing.T) {
 		err = setupCommonStub(context.Background(), cfg, logr.Discard(), "", invalidKubeconfigFile)
 		assert.Nil(t, err)
 	})
+
+	t.Run("should initialize metrics and kube client", func(t *testing.T) {
+		cfg := &controller.ImageUpdaterConfig{}
+		err := callSetupCommonWithMocks(t, cfg, logr.Discard(), "", kubeconfigFile)
+		require.NoError(t, err)
+		assert.NotNil(t, metrics.Endpoint())
+		assert.NotNil(t, metrics.Applications())
+		assert.NotNil(t, metrics.Clients())
+		assert.NotNil(t, cfg.KubeClient)
+		assert.IsType(t, &aiukube.ImageUpdaterKubernetesClient{}, cfg.KubeClient)
+	})
 }

cmd/run.go

@@ -74,6 +74,13 @@ This enables a CRD-driven approach to automated image updates with Argo CD.
 			default:
 				return fmt.Errorf("invalid log format '%s'", cfg.LogFormat)
 			}
+
+			if once {
+				cfg.CheckInterval = 0
+				probeAddr = "0"
+				warmUpCache = true
+			}
+
 			log.SetLogFormat(logFormat)
 
 			ctrl.SetLogger(logrusr.New(log.Log()))
@@ -85,15 +92,9 @@ This enables a CRD-driven approach to automated image updates with Argo CD.
 				"app", version.BinaryName()+": "+version.Version(),
 				"loglevel", strings.ToUpper(cfg.LogLevel),
 				"interval", argocd.GetPrintableInterval(cfg.CheckInterval),
-				"healthPort", argocd.GetPrintableHealthPort(cfg.HealthPort),
+				"healthPort", probeAddr,
 			)
 
-			if once {
-				cfg.CheckInterval = 0
-				cfg.HealthPort = 0
-				warmUpCache = true
-			}
-
 			// Create context with signal handling
 			ctx := ctrl.SetupSignalHandler()
 			err := SetupCommon(ctx, cfg, setupLogger, commitMessagePath, kubeConfig)
@@ -152,18 +153,6 @@ This enables a CRD-driven approach to automated image updates with Argo CD.
 				LeaderElection:          enableLeaderElection,
 				LeaderElectionID:        "c21b75f2.argoproj.io",
 				LeaderElectionNamespace: leaderElectionNamespace,
-
-				// LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily
-				// when the Manager ends. This requires the binary to immediately end when the
-				// Manager is stopped, otherwise, this setting is unsafe. Setting this significantly
-				// speeds up voluntary leader transitions as the new leader don't have to wait
-				// LeaseDuration time first.
-				//
-				// In the default scaffold provided, the program ends immediately after
-				// the manager stops, so would be fine to enable this option. However,
-				// if you are doing or is intended to do any operation such as perform cleanups
-				// after the manager stops then its usage might be unsafe.
-				// LeaderElectionReleaseOnCancel: true,
 			})
 			if err != nil {
 				setupLogger.Error(err, "unable to start manager")
@@ -201,6 +190,7 @@ This enables a CRD-driven approach to automated image updates with Argo CD.
 				setupLogger.Info("Cache warm-up disabled, skipping cache warmer")
 				// If warm-up is disabled, we need to signal that cache is warmed
 				close(warmupState.Done)
+				warmupState.isCacheWarmed.Store(true)
 			}
 
 			// Start the webhook server if enabled
@@ -267,16 +257,11 @@ This enables a CRD-driven approach to automated image updates with Argo CD.
 		},
 	}
 
-	// TODO: flags below are not documented yet and don't have env vars yet. Metrics and health checks will be implemented in GITOPS-7113
 	controllerCmd.Flags().StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
-	controllerCmd.Flags().StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
+	controllerCmd.Flags().StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to. Leave as 0 to disable the probe service.")
 	controllerCmd.Flags().BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
 	controllerCmd.Flags().BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers")
 
-	// TODO: most probably legacy flags. Will be checked in GITOPS-7113
-	controllerCmd.Flags().IntVar(&cfg.HealthPort, "health-port", 8080, "port to start the health server on, 0 to disable")
-	controllerCmd.Flags().IntVar(&cfg.MetricsPort, "metrics-port", 8081, "port to start the metrics server on, 0 to disable")
-
 	controllerCmd.Flags().BoolVar(&enableLeaderElection, "leader-election", true, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
 	controllerCmd.Flags().StringVar(&leaderElectionNamespace, "leader-election-namespace", "", "The namespace used for the leader election lease. If empty, the controller will use the namespace of the pod it is running in. When running locally this value must be set.")
 	controllerCmd.Flags().BoolVar(&cfg.DryRun, "dry-run", false, "run in dry-run mode. If set to true, do not perform any changes")
@@ -285,21 +270,23 @@ This enables a CRD-driven approach to automated image updates with Argo CD.
 	controllerCmd.Flags().StringVar(&cfg.LogFormat, "logformat", env.GetStringVal("IMAGE_UPDATER_LOGFORMAT", "text"), "set the log format to one of text|json")
 	controllerCmd.Flags().StringVar(&kubeConfig, "kubeconfig", "", "full path to kubernetes client configuration, i.e. ~/.kube/config")
 
-	controllerCmd.Flags().BoolVar(&once, "once", false, "run only once, same as specifying --warmup-cache=true, --interval=0 and --health-port=0")
+	controllerCmd.Flags().BoolVar(&once, "once", false, "run only once, same as specifying --warmup-cache=true, --interval=0 and --health-probe-bind-address=0")
 	controllerCmd.Flags().StringVar(&cfg.RegistriesConf, "registries-conf-path", common.DefaultRegistriesConfPath, "path to registries configuration file")
 	controllerCmd.Flags().IntVar(&cfg.MaxConcurrentApps, "max-concurrent-apps", env.ParseNumFromEnv("MAX_CONCURRENT_APPS", 10, 1, 100), "maximum number of ArgoCD applications that can be updated concurrently (must be >= 1)")
 	controllerCmd.Flags().IntVar(&MaxConcurrentReconciles, "max-concurrent-reconciles", env.ParseNumFromEnv("MAX_CONCURRENT_RECONCILES", 1, 1, 10), "maximum number of concurrent Reconciles which can be run (must be >= 1)")
 	controllerCmd.Flags().StringVar(&cfg.ArgocdNamespace, "argocd-namespace", "", "namespace where ArgoCD runs in (current namespace by default)")
 	controllerCmd.Flags().BoolVar(&warmUpCache, "warmup-cache", true, "whether to perform a cache warm-up on startup")
+	controllerCmd.Flags().BoolVar(&cfg.DisableKubeEvents, "disable-kube-events", env.GetBoolVal("IMAGE_UPDATER_KUBE_EVENTS", false), "Disable kubernetes events")
 
+	// Git flags
 	controllerCmd.Flags().StringVar(&cfg.GitCommitUser, "git-commit-user", env.GetStringVal("GIT_COMMIT_USER", "argocd-image-updater"), "Username to use for Git commits")
 	controllerCmd.Flags().StringVar(&cfg.GitCommitMail, "git-commit-email", env.GetStringVal("GIT_COMMIT_EMAIL", "[email protected]"), "E-Mail address to use for Git commits")
 	controllerCmd.Flags().StringVar(&cfg.GitCommitSigningKey, "git-commit-signing-key", env.GetStringVal("GIT_COMMIT_SIGNING_KEY", ""), "GnuPG key ID or path to Private SSH Key used to sign the commits")
 	controllerCmd.Flags().StringVar(&cfg.GitCommitSigningMethod, "git-commit-signing-method", env.GetStringVal("GIT_COMMIT_SIGNING_METHOD", "openpgp"), "Method used to sign Git commits ('openpgp' or 'ssh')")
 	controllerCmd.Flags().BoolVar(&cfg.GitCommitSignOff, "git-commit-sign-off", env.GetBoolVal("GIT_COMMIT_SIGN_OFF", false), "Whether to sign-off git commits")
 	controllerCmd.Flags().StringVar(&commitMessagePath, "git-commit-message-path", common.DefaultCommitTemplatePath, "Path to a template to use for Git commit messages")
-	controllerCmd.Flags().BoolVar(&cfg.DisableKubeEvents, "disable-kube-events", env.GetBoolVal("IMAGE_UPDATER_KUBE_EVENTS", false), "Disable kubernetes events")
 
+	// Webhook flags
 	controllerCmd.Flags().BoolVar(&cfg.EnableWebhook, "enable-webhook", env.GetBoolVal("ENABLE_WEBHOOK", false), "Enable webhook server for receiving registry events")
 	controllerCmd.Flags().IntVar(&webhookCfg.Port, "webhook-port", env.ParseNumFromEnv("WEBHOOK_PORT", 8082, 0, 65535), "Port to listen on for webhook events")
 	controllerCmd.Flags().StringVar(&webhookCfg.DockerSecret, "docker-webhook-secret", env.GetStringVal("DOCKER_WEBHOOK_SECRET", ""), "Secret for validating Docker Hub webhooks")

cmd/run_test.go

@@ -2,7 +2,9 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"math"
+	"net/http"
 	"os"
 	"strconv"
 	"testing"
@@ -43,8 +45,6 @@ func TestNewRunCommand(t *testing.T) {
 	asser.Equal(env.GetStringVal("IMAGE_UPDATER_LOGLEVEL", "info"), controllerCommand.Flag("loglevel").Value.String())
 	asser.Equal(env.GetStringVal("IMAGE_UPDATER_LOGFORMAT", "text"), controllerCommand.Flag("logformat").Value.String())
 	asser.Equal("", controllerCommand.Flag("kubeconfig").Value.String())
-	asser.Equal("8080", controllerCommand.Flag("health-port").Value.String())
-	asser.Equal("8081", controllerCommand.Flag("metrics-port").Value.String())
 	asser.Equal("false", controllerCommand.Flag("once").Value.String())
 	asser.Equal(common.DefaultRegistriesConfPath, controllerCommand.Flag("registries-conf-path").Value.String())
 	asser.Equal(strconv.Itoa(env.ParseNumFromEnv("MAX_CONCURRENT_APPS", 10, 1, 100)), controllerCommand.Flag("max-concurrent-apps").Value.String())
@@ -297,3 +297,46 @@ func TestWebhookServerRunnable_Start_ContextCancelStopsServer(t *testing.T) {
 		assert.NotNil(t, ws.webhookServer.Server)
 	}
 }
+
+// Assisted-by: Gemini AI
+// TestReadyzCheckWithWarmupStatus verifies the behavior of the warmup-check readiness probe
+// based on the state of WarmupStatus.
+func TestReadyzCheckWithWarmupStatus(t *testing.T) {
+	// Sub-test for when cache warm-up is disabled
+	t.Run("warmup-disabled", func(t *testing.T) {
+		status := &WarmupStatus{Done: make(chan struct{})}
+
+		// Simulate the logic for when warmUpCache is false
+		status.isCacheWarmed.Store(true)
+
+		// Create a fake readiness check that uses the warmup status
+		check := func(req *http.Request) error {
+			if !status.isCacheWarmed.Load() {
+				return fmt.Errorf("cache is not yet warmed")
+			}
+			return nil
+		}
+
+		// The check should pass because isCacheWarmed is true
+		err := check(nil)
+		assert.NoError(t, err, "readiness check should pass when cache warmup is disabled")
+	})
+
+	// Sub-test for when cache warm-up is enabled but not yet complete
+	t.Run("warmup-enabled-not-warmed", func(t *testing.T) {
+		status := &WarmupStatus{Done: make(chan struct{})}
+
+		// In this case, isCacheWarmed is still false
+		check := func(req *http.Request) error {
+			if !status.isCacheWarmed.Load() {
+				return fmt.Errorf("cache is not yet warmed")
+			}
+			return nil
+		}
+
+		// The check should fail because isCacheWarmed is false
+		err := check(nil)
+		assert.Error(t, err, "readiness check should fail when cache is not warmed")
+		assert.Equal(t, "cache is not yet warmed", err.Error())
+	})
+}

config/default/kustomization.yaml

@@ -22,20 +22,20 @@ resources:
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 #- ../prometheus
 # [METRICS] Expose the controller manager metrics service.
-#- metrics_service.yaml
+- metrics_service.yaml
 # [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy.
 # Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics.
 # Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will
 # be able to communicate with the Webhook Server.
-#- ../network-policy
+- ../network-policy
 
 # Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager
-#patches:
+patches:
 # [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443.
 # More info: https://book.kubebuilder.io/reference/metrics
-#- path: manager_metrics_patch.yaml
-#  target:
-#    kind: Deployment
+- path: manager_metrics_patch.yaml
+  target:
+    kind: Deployment
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml

config/default/metrics_service.yaml

@@ -6,7 +6,7 @@ metadata:
     app.kubernetes.io/name: argocd-image-updater
     app.kubernetes.io/managed-by: kustomize
   name: argocd-image-updater-controller-metrics-service
-  namespace: system
+  namespace: argocd-image-updater-system
 spec:
   ports:
   - name: https

config/install.yaml

@@ -5,6 +5,7 @@ metadata:
     app.kubernetes.io/managed-by: kustomize
     app.kubernetes.io/name: argocd-image-updater-system
     control-plane: argocd-image-updater-controller
+    metrics: enabled
   name: argocd-image-updater-system
 ---
 apiVersion: apiextensions.k8s.io/v1
@@ -661,6 +662,34 @@ rules:
   - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: argocd-image-updater-metrics-auth-role
+rules:
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: argocd-image-updater-metrics-reader
+rules:
+- nonResourceURLs:
+  - /metrics
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   labels:
@@ -711,6 +740,32 @@ subjects:
   name: argocd-image-updater-controller
   namespace: argocd-image-updater-system
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: argocd-image-updater-metrics-auth-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: argocd-image-updater-metrics-auth-role
+subjects:
+- kind: ServiceAccount
+  name: argocd-image-updater-controller
+  namespace: argocd-image-updater-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: argocd-image-updater-metrics-reader-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: argocd-image-updater-metrics-reader
+subjects:
+- kind: ServiceAccount
+  name: argocd-image-updater-controller
+  namespace: argocd-image-updater-system
+---
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -738,6 +793,24 @@ metadata:
   name: argocd-image-updater-secret
   namespace: argocd-image-updater-system
 ---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/name: argocd-image-updater
+    control-plane: argocd-image-updater-controller
+  name: argocd-image-updater-controller-metrics-service
+  namespace: argocd-image-updater-system
+spec:
+  ports:
+  - name: https
+    port: 8443
+    protocol: TCP
+    targetPort: 8443
+  selector:
+    control-plane: argocd-image-updater-controller
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -762,6 +835,7 @@ spec:
     spec:
       containers:
       - args:
+        - --metrics-bind-address=:8443
         - run
         command:
         - /manager
@@ -945,3 +1019,27 @@ spec:
           secretName: ssh-git-creds
       - emptyDir: {}
         name: tmp
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/name: argocd-image-updater-controller
+    control-plane: argocd-image-updater-controller
+  name: allow-metrics-traffic
+  namespace: argocd-image-updater-system
+spec:
+  ingress:
+  - from:
+    - namespaceSelector:
+        matchLabels:
+          metrics: enabled
+    ports:
+    - port: 8443
+      protocol: TCP
+  podSelector:
+    matchLabels:
+      control-plane: argocd-image-updater-controller
+  policyTypes:
+  - Ingress

config/manager/manager.yaml

@@ -5,6 +5,7 @@ metadata:
     control-plane: argocd-image-updater-controller
     app.kubernetes.io/name: argocd-image-updater-system
     app.kubernetes.io/managed-by: kustomize
+    metrics: enabled
   name: argocd-image-updater-system
 ---
 apiVersion: apps/v1