Add automatic_updates label (#40563)

lib/auth/auth.go

@@ -686,9 +686,12 @@ var (
 		prometheus.GaugeOpts{
 			Namespace: teleport.MetricNamespace,
 			Name:      teleport.MetricRegisteredServers,
-			Help:      "The number of Teleport services that are connected to an auth server by version.",
+			Help:      "The number of Teleport services that are connected to an auth server.",
+		},
+		[]string{
+			teleport.TagVersion,
+			teleport.TagAutomaticUpdates,
 		},
-		[]string{teleport.TagVersion},
 	)
 
 	registeredAgentsInstallMethod = prometheus.NewGaugeVec(
@@ -1304,8 +1307,7 @@ func (a *Server) runPeriodicOperations() {
 			// Update prometheus gauge
 			heartbeatsMissedByAuth.Set(float64(missedKeepAliveCount))
 		case <-promTicker.Next():
-			a.updateVersionMetrics()
-			a.updateInstallMethodsMetrics()
+			a.updateAgentMetrics()
 		case <-releaseCheck.Next():
 			a.syncReleaseAlerts(ctx, true)
 		case <-localReleaseCheck.Next():
@@ -1376,8 +1378,6 @@ func (a *Server) doInstancePeriodics(ctx context.Context) {
 		return
 	}
 
-	a.updateUpdaterVersionMetrics()
-
 	// create/delete upgrade enroll prompt as appropriate
 	enrollMsg, shouldPrompt := uep.GenerateEnrollPrompt()
 	a.handleUpgradeEnrollPrompt(ctx, enrollMsg, shouldPrompt)
@@ -1563,76 +1563,39 @@ func (a *Server) doReleaseAlertSync(ctx context.Context, current vc.Target, visi
 	}
 }
 
-// updateUpdaterVersionMetrics leverages the inventory control stream to report the
-// number of teleport updaters installed and their versions. To get an accurate representation
-// of versions in an entire cluster the metric must be aggregated with all auth instances.
-func (a *Server) updateUpdaterVersionMetrics() {
+func (a *Server) updateAgentMetrics() {
 	imp := newInstanceMetricsPeriodic()
 
-	// record versions for all connected resources
 	a.inventory.Iter(func(handle inventory.UpstreamHandle) {
-		imp.VisitInstance(handle.Hello())
+		imp.VisitInstance(handle.Hello(), handle.AgentMetadata())
 	})
 
 	totalInstancesMetric.Set(float64(imp.TotalInstances()))
 	enrolledInUpgradesMetric.Set(float64(imp.TotalEnrolledInUpgrades()))
 
-	// reset the gauges so that any versions that fall off are removed from exported metrics
-	upgraderCountsMetric.Reset()
-	for upgraderType, upgraderVersions := range imp.upgraderCounts {
-		for version, count := range upgraderVersions {
-			upgraderCountsMetric.With(prometheus.Labels{
-				teleport.TagUpgrader: upgraderType,
-				teleport.TagVersion:  version,
-			}).Set(float64(count))
-		}
-	}
-}
-
-// updateVersionMetrics leverages the inventory control stream to report the versions of
-// all instances that are connected to a single auth server via prometheus metrics. To
-// get an accurate representation of versions in an entire cluster the metric must be aggregated
-// with all auth instances.
-func (a *Server) updateVersionMetrics() {
-	versionCount := make(map[string]int)
-
-	// record versions for all connected resources
-	a.inventory.Iter(func(handle inventory.UpstreamHandle) {
-		versionCount[handle.Hello().Version]++
-	})
-
 	// reset the gauges so that any versions that fall off are removed from exported metrics
 	registeredAgents.Reset()
-	for version, count := range versionCount {
-		registeredAgents.WithLabelValues(version).Set(float64(count))
+	for agent, count := range imp.RegisteredAgentsCount() {
+		registeredAgents.With(prometheus.Labels{
+			teleport.TagVersion:          agent.version,
+			teleport.TagAutomaticUpdates: agent.automaticUpdates,
+		}).Set(float64(count))
 	}
-}
-
-// updateInstallMethodsMetrics leverages the inventory control stream to report the install methods
-// of all instances that are connected to a single auth server via prometheus metrics.
-// To get an accurate representation of install methods in an entire cluster the metric must be aggregated
-// with all auth instances.
-func (a *Server) updateInstallMethodsMetrics() {
-	installMethodCount := make(map[string]int)
-
-	// record install methods for all connected resources
-	a.inventory.Iter(func(handle inventory.UpstreamHandle) {
-		installMethod := "unknown"
-		installMethods := append([]string{}, handle.AgentMetadata().InstallMethods...)
-
-		if len(installMethods) > 0 {
-			slices.Sort(installMethods)
-			installMethod = strings.Join(installMethods, ",")
-		}
-
-		installMethodCount[installMethod]++
-	})
 
 	// reset the gauges so that any versions that fall off are removed from exported metrics
 	registeredAgentsInstallMethod.Reset()
-	for installMethod, count := range installMethodCount {
+	for installMethod, count := range imp.InstallMethodCounts() {
 		registeredAgentsInstallMethod.WithLabelValues(installMethod).Set(float64(count))
 	}
+
+	// reset the gauges so that any type+version that fall off are removed from exported metrics
+	upgraderCountsMetric.Reset()
+	for metadata, count := range imp.UpgraderCounts() {
+		upgraderCountsMetric.With(prometheus.Labels{
+			teleport.TagUpgrader: metadata.upgraderType,
+			teleport.TagVersion:  metadata.version,
+		}).Set(float64(count))
+	}
 }
 
 var (

lib/auth/periodic.go

@@ -20,6 +20,8 @@ package auth
 
 import (
 	"fmt"
+	"slices"
+	"strings"
 
 	"golang.org/x/mod/semver"
 
@@ -104,40 +106,112 @@ func inspectVersionCounts(counts map[string]int) (median string, total int, ok b
 
 // instanceMetricsPeriodic is an aggregator for general instance metrics.
 type instanceMetricsPeriodic struct {
-	upgraderCounts map[string]map[string]int
-	totalInstances int
+	metadata []instanceMetadata
+}
+
+// instanceMetadata contains instance metadata to be exported.
+type instanceMetadata struct {
+	// version specifies the version of the Teleport instance
+	version string
+	// installMethod specifies the Teleport agent installation method
+	installMethod string
+	// upgraderType specifies the upgrader type
+	upgraderType string
+	// upgraderVersion specifies the upgrader version
+	upgraderVersion string
 }
 
 func newInstanceMetricsPeriodic() *instanceMetricsPeriodic {
 	return &instanceMetricsPeriodic{
-		upgraderCounts: make(map[string]map[string]int),
+		metadata: []instanceMetadata{},
 	}
 }
 
-// VisitInstance adds an instance to ongoing aggregations.
-func (i *instanceMetricsPeriodic) VisitInstance(instance proto.UpstreamInventoryHello) {
-	i.totalInstances++
-	if upgrader := instance.GetExternalUpgrader(); upgrader != "" {
-		if _, exists := i.upgraderCounts[upgrader]; !exists {
-			i.upgraderCounts[upgrader] = make(map[string]int)
+func (i *instanceMetricsPeriodic) VisitInstance(instance proto.UpstreamInventoryHello, metadata proto.UpstreamInventoryAgentMetadata) {
+	// Sort install methods if multiple methods are specified.
+	installMethod := "unknown"
+	installMethods := append([]string{}, metadata.GetInstallMethods()...)
+	if len(installMethods) > 0 {
+		slices.Sort(installMethods)
+		installMethod = strings.Join(installMethods, ",")
+	}
+
+	iMetadata := instanceMetadata{
+		version:         instance.GetVersion(),
+		installMethod:   installMethod,
+		upgraderType:    instance.GetExternalUpgrader(),
+		upgraderVersion: instance.GetExternalUpgraderVersion(),
+	}
+	i.metadata = append(i.metadata, iMetadata)
+}
+
+type registeredAgent struct {
+	version          string
+	automaticUpdates string
+}
+
+// RegisteredAgentsCount returns the count registered agents count.
+func (i *instanceMetricsPeriodic) RegisteredAgentsCount() map[registeredAgent]int {
+	result := make(map[registeredAgent]int)
+	for _, metadata := range i.metadata {
+		automaticUpdates := "false"
+		if metadata.upgraderType != "" {
+			automaticUpdates = "true"
+		}
+
+		agent := registeredAgent{
+			version:          metadata.version,
+			automaticUpdates: automaticUpdates,
 		}
-		i.upgraderCounts[upgrader][instance.GetExternalUpgraderVersion()]++
+		result[agent]++
 	}
+	return result
+}
+
+// InstallMethodCounts returns the count of each install method.
+func (i *instanceMetricsPeriodic) InstallMethodCounts() map[string]int {
+	installMethodCount := make(map[string]int)
+	for _, metadata := range i.metadata {
+		installMethodCount[metadata.installMethod]++
+	}
+	return installMethodCount
+}
+
+type upgrader struct {
+	upgraderType string
+	version      string
+}
+
+// UpgraderCounts returns the count for the different upgrader version and type combinations.
+func (i *instanceMetricsPeriodic) UpgraderCounts() map[upgrader]int {
+	result := make(map[upgrader]int)
+	for _, metadata := range i.metadata {
+		// Do not count the instance if a type is not specified
+		if metadata.upgraderType == "" {
+			continue
+		}
+
+		upgrader := upgrader{
+			upgraderType: metadata.upgraderType,
+			version:      metadata.upgraderVersion,
+		}
+		result[upgrader]++
+	}
+	return result
 }
 
 // TotalEnrolledInUpgrades gets the total number of instances that have some upgrader defined.
 func (i *instanceMetricsPeriodic) TotalEnrolledInUpgrades() int {
 	var total int
-	for _, upgraderVersion := range i.upgraderCounts {
-		for _, count := range upgraderVersion {
-			total += count
+	for _, metadata := range i.metadata {
+		if metadata.upgraderType != "" {
+			total++
 		}
 	}
-
 	return total
 }
 
 // TotalInstances gets the total number of known instances.
 func (i *instanceMetricsPeriodic) TotalInstances() int {
-	return i.totalInstances
+	return len(i.metadata)
 }