feat(metrics): add ARC domain metrics

api/arc/v1alpha1/labels.go

@@ -0,0 +1,12 @@
+// Copyright 2025 BWI GmbH and Artifact Conduit contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+const (
+	// LabelArtifactType records which ArtifactType an ArtifactWorkflow was
+	// derived from. The type is only present on the owning Order, so the
+	// controller stamps it here to make workflows selectable and observable
+	// by type.
+	LabelArtifactType = "arc.opendefense.cloud/artifact-type"
+)

cmd/arc-controller-manager/main.go

@@ -21,11 +21,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/certwatcher"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	arcv1alpha1 "go.opendefense.cloud/arc/api/arc/v1alpha1"
 	"go.opendefense.cloud/arc/pkg/controller"
+	arcmetrics "go.opendefense.cloud/arc/pkg/metrics"
 
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 )
@@ -168,6 +170,14 @@ func main() {
 		}
 	}
 
+	arcMetrics := arcmetrics.NewCollector(mgr.GetCache())
+	ctrlmetrics.Registry.MustRegister(arcMetrics)
+
+	if err := mgr.Add(arcMetrics); err != nil {
+		setupLog.Error(err, "unable to add metrics leader gate")
+		os.Exit(1)
+	}
+
 	if err := wfv1alpha1.AddToScheme(mgr.GetScheme()); err != nil {
 		setupLog.Error(err, "failed to add Argo Workflows types to scheme")
 		os.Exit(1)

go.mod

@@ -8,6 +8,8 @@ require (
 	github.com/jastBytes/sprint v0.0.3
 	github.com/onsi/ginkgo/v2 v2.32.1
 	github.com/onsi/gomega v1.42.1
+	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/client_model v0.6.2
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/spf13/pflag v1.0.10
 	go.opendefense.cloud/kit v0.3.4
@@ -76,8 +78,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.23.2 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.70.1 // indirect
 	github.com/prometheus/procfs v0.21.0 // indirect
 	github.com/spf13/cobra v1.10.2 // indirect

pkg/controller/artifactworkflow_controller.go

@@ -28,6 +28,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	arcv1alpha1 "go.opendefense.cloud/arc/api/arc/v1alpha1"
+	"go.opendefense.cloud/arc/pkg/metrics"
 )
 
 const (
@@ -146,10 +147,11 @@ func (r *ArtifactWorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	return ctrlResult, nil
 }
 
-func (r *ArtifactWorkflowReconciler) setStatusFromWorkflow(ctx context.Context, log logr.Logger, aw *arcv1alpha1.ArtifactWorkflow, wf *wfv1alpha1.Workflow) bool {
+func (r *ArtifactWorkflowReconciler) setStatusFromWorkflow(ctx context.Context, log logr.Logger, aw *arcv1alpha1.ArtifactWorkflow, wf *wfv1alpha1.Workflow) (bool, *completion) {
 	if aw.Status.Phase == arcv1alpha1.WorkflowPhase(wf.Status.Phase) {
-		return false // nothing updated
+		return false, nil // nothing updated
 	}
+
 	aw.Status.Phase = arcv1alpha1.WorkflowPhase(wf.Status.Phase)
 
 	switch aw.Status.Phase {
@@ -162,7 +164,7 @@ func (r *ArtifactWorkflowReconciler) setStatusFromWorkflow(ctx context.Context,
 	default:
 	}
 
-	return true
+	return true, newCompletion(aw, wf)
 }
 
 func (r *ArtifactWorkflowReconciler) generateWorkflowStatusMessage(ctx context.Context, wf *wfv1alpha1.Workflow, log logr.Logger, aw *arcv1alpha1.ArtifactWorkflow) {
@@ -224,15 +226,19 @@ func (r *ArtifactWorkflowReconciler) retrieveSecrets(ctx context.Context, aw *ar
 	srcSecret := corev1.Secret{}
 	if aw.Spec.SrcSecretRef.Name != "" {
 		if err := r.Get(ctx, namespacedName(aw.Namespace, aw.Spec.SrcSecretRef.Name), &srcSecret); err != nil {
-			r.Recorder.Eventf(aw, nil, corev1.EventTypeWarning, "InvalidSecret", "FetchSecret", fmt.Sprintf("Failed to fetch source secret '%s': %v", aw.Spec.SrcSecretRef.Name, err))
+			r.Recorder.Eventf(aw, nil, corev1.EventTypeWarning, ReasonInvalidSecret, "FetchSecret", fmt.Sprintf("Failed to fetch source secret '%s': %v", aw.Spec.SrcSecretRef.Name, err))
+			metrics.RecordReconcileError(ControllerArtifactWorkflow, ReasonInvalidSecret)
+
 			return nil, nil, fmt.Errorf("failed to fetch secret for source: %w", err)
 		}
 	}
 
 	dstSecret := corev1.Secret{}
 	if aw.Spec.DstSecretRef.Name != "" {
 		if err := r.Get(ctx, namespacedName(aw.Namespace, aw.Spec.DstSecretRef.Name), &dstSecret); err != nil {
-			r.Recorder.Eventf(aw, nil, corev1.EventTypeWarning, "InvalidSecret", "FetchSecret", fmt.Sprintf("Failed to fetch destination secret '%s': %v", aw.Spec.DstSecretRef.Name, err))
+			r.Recorder.Eventf(aw, nil, corev1.EventTypeWarning, ReasonInvalidSecret, "FetchSecret", fmt.Sprintf("Failed to fetch destination secret '%s': %v", aw.Spec.DstSecretRef.Name, err))
+			metrics.RecordReconcileError(ControllerArtifactWorkflow, ReasonInvalidSecret)
+
 			return nil, nil, fmt.Errorf("failed to fetch secret for destination: %w", err)
 		}
 	}

pkg/controller/artifactworkflow_controller_test.go

@@ -7,13 +7,15 @@ import (
 	"fmt"
 
 	wfv1alpha1 "github.com/argoproj/argo-workflows/v4/pkg/apis/workflow/v1alpha1"
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"go.opendefense.cloud/kit/envtest"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/scheme"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	arcv1alpha1 "go.opendefense.cloud/arc/api/arc/v1alpha1"
+	"go.opendefense.cloud/arc/pkg/metrics"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -122,6 +124,32 @@ var _ = Describe("ArtifactWorkflowController", func() {
 			}))
 		})
 
+		It("should count a missing secret under the reason its Event carries", func() {
+			counter := metrics.ReconcileErrorsCounterForTest(ControllerArtifactWorkflow, ReasonInvalidSecret)
+			before := testutil.ToFloat64(counter)
+
+			aw := &arcv1alpha1.ArtifactWorkflow{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: ns.Name,
+					Name:      "missing-secret",
+				},
+				Spec: arcv1alpha1.ArtifactWorkflowSpec{
+					WorkflowTemplateRef: at.Spec.WorkflowTemplateRef,
+					SrcSecretRef:        corev1.LocalObjectReference{Name: "does-not-exist"},
+				},
+			}
+			Expect(k8sClient.Create(ctx, aw)).To(Succeed())
+
+			Eventually(func() float64 {
+				return testutil.ToFloat64(counter) - before
+			}).Should(BeNumerically(">=", 1.0))
+
+			// The workflow must not be created from secrets that could not be read.
+			Consistently(func() error {
+				return k8sClient.Get(ctx, namespacedName(ns.Name, aw.Name), &wfv1alpha1.Workflow{})
+			}).ShouldNot(Succeed())
+		})
+
 		It("should track Workflow status changes of created ArtifactWorkflows", func() {
 			awName := "track-status"
 			aw := &arcv1alpha1.ArtifactWorkflow{
@@ -167,6 +195,40 @@ var _ = Describe("ArtifactWorkflowController", func() {
 			}).Should(Equal(int64(1)))
 		})
 
+		It("should count a completion once the workflow succeeds", func() {
+			counter := metrics.CompletionsCounterForTest(ns.Name, metrics.UnknownArtifactType, metrics.ResultSucceeded)
+			before := testutil.ToFloat64(counter)
+
+			awName := "count-completion"
+			aw := &arcv1alpha1.ArtifactWorkflow{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: ns.Name,
+					Name:      awName,
+				},
+				Spec: arcv1alpha1.ArtifactWorkflowSpec{
+					WorkflowTemplateRef: at.Spec.WorkflowTemplateRef,
+					Parameters: []arcv1alpha1.ArtifactWorkflowParameter{
+						{Name: awName, Value: awName},
+					},
+				},
+			}
+			Expect(k8sClient.Create(ctx, aw)).To(Succeed())
+
+			wf := &wfv1alpha1.Workflow{}
+			Eventually(func() error {
+				return k8sClient.Get(ctx, namespacedName(aw.Namespace, aw.Name), wf)
+			}).Should(Succeed())
+
+			wf.Status.Phase = wfv1alpha1.WorkflowSucceeded
+			Expect(k8sClient.Update(ctx, wf)).To(Succeed())
+
+			delta := func() float64 {
+				return testutil.ToFloat64(counter) - before
+			}
+			Eventually(delta).Should(Equal(1.0))
+			Consistently(delta).Should(Equal(1.0))
+		})
+
 		It("should track failed Workflow information of created ArtifactWorkflows", func() {
 			awName := "track-failed-status"
 			aw := &arcv1alpha1.ArtifactWorkflow{
@@ -504,3 +566,94 @@ var _ = Describe("ArtifactWorkflowController", func() {
 		})
 	})
 })
+
+var _ = Describe("newCompletion", func() {
+	It("should return nil for non terminal phases", func() {
+		aw := &arcv1alpha1.ArtifactWorkflow{}
+		aw.Status.Phase = arcv1alpha1.WorkflowRunning
+
+		Expect(newCompletion(aw, &wfv1alpha1.Workflow{})).To(BeNil())
+	})
+
+	It("should return nil for Stopped, which is an action not a result", func() {
+		aw := &arcv1alpha1.ArtifactWorkflow{}
+		aw.Status.Phase = arcv1alpha1.WorkflowStopped
+
+		Expect(newCompletion(aw, &wfv1alpha1.Workflow{})).To(BeNil())
+	})
+
+	It("should take the duration from the argo workflow", func() {
+		aw := &arcv1alpha1.ArtifactWorkflow{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: "team-a",
+				Labels:    map[string]string{arcv1alpha1.LabelArtifactType: "oci"},
+			},
+		}
+		aw.Status.Phase = arcv1alpha1.WorkflowSucceeded
+
+		wf := &wfv1alpha1.Workflow{}
+		wf.Status.StartedAt = metav1.NewTime(metav1.Unix(1700000000, 0).Time)
+		wf.Status.FinishedAt = metav1.NewTime(metav1.Unix(1700000090, 0).Time)
+
+		completion := newCompletion(aw, wf)
+
+		Expect(completion).NotTo(BeNil())
+		Expect(completion.namespace).To(Equal("team-a"))
+		Expect(completion.artifactType).To(Equal("oci"))
+		Expect(completion.result).To(Equal(metrics.ResultSucceeded))
+		Expect(completion.hasDuration).To(BeTrue())
+		Expect(completion.seconds).To(Equal(90.0))
+	})
+
+	It("should record a zero length duration when start and finish share the same second", func() {
+		aw := &arcv1alpha1.ArtifactWorkflow{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: "team-a",
+				Labels:    map[string]string{arcv1alpha1.LabelArtifactType: "oci"},
+			},
+		}
+		aw.Status.Phase = arcv1alpha1.WorkflowSucceeded
+
+		same := metav1.NewTime(metav1.Unix(1700000000, 0).Time)
+		wf := &wfv1alpha1.Workflow{}
+		wf.Status.StartedAt = same
+		wf.Status.FinishedAt = same
+
+		completion := newCompletion(aw, wf)
+
+		Expect(completion).NotTo(BeNil())
+		Expect(completion.hasDuration).To(BeTrue())
+		Expect(completion.seconds).To(Equal(0.0))
+	})
+
+	It("should not record a duration when finish precedes start", func() {
+		aw := &arcv1alpha1.ArtifactWorkflow{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: "team-a",
+				Labels:    map[string]string{arcv1alpha1.LabelArtifactType: "oci"},
+			},
+		}
+		aw.Status.Phase = arcv1alpha1.WorkflowSucceeded
+
+		wf := &wfv1alpha1.Workflow{}
+		wf.Status.StartedAt = metav1.NewTime(metav1.Unix(1700000090, 0).Time)
+		wf.Status.FinishedAt = metav1.NewTime(metav1.Unix(1700000000, 0).Time)
+
+		completion := newCompletion(aw, wf)
+
+		Expect(completion).NotTo(BeNil())
+		Expect(completion.hasDuration).To(BeFalse())
+	})
+
+	It("should record a failure without a duration when argo has no timestamps", func() {
+		aw := &arcv1alpha1.ArtifactWorkflow{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a"}}
+		aw.Status.Phase = arcv1alpha1.WorkflowFailed
+
+		completion := newCompletion(aw, &wfv1alpha1.Workflow{})
+
+		Expect(completion).NotTo(BeNil())
+		Expect(completion.result).To(Equal(metrics.ResultFailed))
+		Expect(completion.artifactType).To(Equal(metrics.UnknownArtifactType))
+		Expect(completion.hasDuration).To(BeFalse())
+	})
+})

pkg/controller/const.go

@@ -7,3 +7,27 @@ const (
 	AnnotationRequestedAt = "arc.opendefense.cloud/requested-at"
 	AnnotationForceAt     = "arc.opendefense.cloud/force-at"
 )
+
+// Event reasons. These double as the reason label on arc_reconcile_errors_total,
+// so the metric and the Kubernetes Event for the same failure always agree.
+const (
+	ReasonInvalid             = "Invalid"
+	ReasonInvalidEndpoint     = "InvalidEndpoint"
+	ReasonInvalidArtifactType = "InvalidArtifactType"
+	ReasonInvalidSecret       = "InvalidSecret"
+	ReasonComputationFailed   = "ComputationFailed"
+	ReasonHydrationFailed     = "HydrationFailed"
+	ReasonCreationFailed      = "CreationFailed"
+	ReasonDeletionFailed      = "DeletionFailed"
+)
+
+// Controller names used as the controller label on arc_reconcile_errors_total.
+const (
+	ControllerOrder            = "order"
+	ControllerArtifactWorkflow = "artifactworkflow"
+)
+
+// ReasonDeleting is the Event reason for the informational warning emitted while
+// an order's deletion is in progress. It is not a failure, so it is not counted
+// on arc_reconcile_errors_total.
+const ReasonDeleting = "Deleting"

pkg/controller/helpers.go

@@ -6,13 +6,15 @@ package controller
 import (
 	"encoding/json"
 	"fmt"
+	"maps"
 	"strconv"
 	"strings"
 	"time"
 
 	wfv1alpha1 "github.com/argoproj/argo-workflows/v4/pkg/apis/workflow/v1alpha1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/validation"
 
 	arcv1alpha1 "go.opendefense.cloud/arc/api/arc/v1alpha1"
 )
@@ -37,12 +39,29 @@ func cloneObjectMeta(meta metav1.ObjectMeta, name string) metav1.ObjectMeta {
 	return metav1.ObjectMeta{
 		Namespace: meta.Namespace,
 		Name:      name,
-		Labels:    meta.Labels,
+		Labels:    maps.Clone(meta.Labels),
 	}
 }
 
-func awObjectMeta(order *arcv1alpha1.Order, sha string) metav1.ObjectMeta {
-	return cloneObjectMeta(order.ObjectMeta, awName(order, sha))
+func awObjectMeta(order *arcv1alpha1.Order, sha, artifactType string) metav1.ObjectMeta {
+	meta := cloneObjectMeta(order.ObjectMeta, awName(order, sha))
+
+	// Nothing validates OrderArtifact.Type against the 63 character label value
+	// limit, and an invalid value would make every create of this
+	// ArtifactWorkflow fail permanently. Leaving the label off keeps the
+	// workflow creatable and reports it as artifact_type="unknown". Truncating
+	// would report a wrong type instead of a missing one.
+	if len(validation.IsValidLabelValue(artifactType)) > 0 {
+		delete(meta.Labels, arcv1alpha1.LabelArtifactType)
+		return meta
+	}
+
+	if meta.Labels == nil {
+		meta.Labels = map[string]string{}
+	}
+	meta.Labels[arcv1alpha1.LabelArtifactType] = artifactType
+
+	return meta
 }
 
 func workflowObjectMeta(aw *arcv1alpha1.ArtifactWorkflow) metav1.ObjectMeta {