feat(otel): add ate.workerpool.workers metric with e2e suite

internal/ateattr/ateattr.go

@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package ateattr projects an Actor onto substrate's ate.* span attributes.
-// Identity is a span-level subject attribute (the producer is the substrate
-// component, the actor is the subject), so it belongs on spans rather than the
-// resource, and uses substrate's own ate.* namespace rather than service.*.
+// Package ateattr is the single source of truth for substrate's ate.* telemetry
+// attributes: the identity keys stamped on spans/logs, and the bounded value
+// sets used as metric labels. Centralizing them keeps a key (and value) meaning
+// the same thing across every signal and binary.
 package ateattr
 
 import (
@@ -43,6 +43,81 @@ const (
 	ActorVersionKey      = attribute.Key("ate.actor.version")
 )
 
+// Metric-label keys: the only ate.* attributes allowed on metric datapoints,
+// each with a small bounded value set. High-cardinality identity (actor
+// name/uid, atespace) is absent by design; it belongs on spans and logs.
+// ActorOperationNameKey follows the registry's *.operation.name pattern
+// (db.operation.name, gen_ai.operation.name). WorkerStateKey stays worker-rooted
+// rather than nesting under the pool so it can grow siblings.
+const (
+	ActorOperationNameKey = attribute.Key("ate.actor.operation.name")
+	WorkerPoolNameKey     = attribute.Key("ate.workerpool.name")
+	WorkerStateKey        = attribute.Key("ate.worker.state")
+	SandboxClassKey       = attribute.Key("ate.sandbox.class")
+	SnapshotKindKey       = attribute.Key("ate.snapshot.kind")
+	SnapshotPhaseKey      = attribute.Key("ate.snapshot.phase")
+	SchedulerOutcomeKey   = attribute.Key("ate.scheduler.outcome")
+)
+
+// ErrorTypeKey is the OTel registry attribute, reused verbatim (not aliased into
+// ate.*): failures are reported on the same instrument via this key, its absence
+// meaning success, never as a parallel _failures counter.
+const ErrorTypeKey = attribute.Key("error.type")
+
+// Values for ActorOperationNameKey.
+const (
+	OperationCreate  = "create"
+	OperationResume  = "resume"
+	OperationSuspend = "suspend"
+	OperationPause   = "pause"
+	OperationDelete  = "delete"
+)
+
+// Values for WorkerStateKey. Only idle and assigned are representable today;
+// starting and unhealthy workers are not modeled in the cache.
+const (
+	WorkerStateIdle     = "idle"
+	WorkerStateAssigned = "assigned"
+)
+
+// Values for SchedulerOutcomeKey. NoFreeWorker is a capacity signal, not an
+// error, so it is a distinct outcome rather than an error.type value.
+const (
+	SchedulerOutcomeAssigned     = "assigned"
+	SchedulerOutcomeNoFreeWorker = "no_free_worker"
+	SchedulerOutcomeError        = "error"
+)
+
+// Values for SnapshotKindKey. Boot is not a restore, so it appears only on the
+// ateapi lifecycle histogram, never on the atelet restore histogram.
+const (
+	SnapshotKindGolden  = "golden"
+	SnapshotKindLatest  = "latest"
+	SnapshotKindBoot    = "boot"
+	SnapshotKindUnknown = "unknown"
+)
+
+// ClampRestoreSnapshotKind bounds an untrusted kind before it becomes a metric
+// label, so a caller cannot inflate cardinality.
+func ClampRestoreSnapshotKind(kind string) string {
+	switch kind {
+	case SnapshotKindGolden, SnapshotKindLatest:
+		return kind
+	default:
+		return SnapshotKindUnknown
+	}
+}
+
+// Values for SnapshotPhaseKey: restore phases (download, oci_unpack,
+// ateom_restore) and checkpoint phases (checkpoint, upload) share the key.
+const (
+	SnapshotPhaseDownload     = "download"
+	SnapshotPhaseOCIUnpack    = "oci_unpack"
+	SnapshotPhaseAteomRestore = "ateom_restore"
+	SnapshotPhaseCheckpoint   = "checkpoint"
+	SnapshotPhaseUpload       = "upload"
+)
+
 // ActorRefAttributes returns the subset knowable before the Actor record
 // resolves: only the (atespace, name) the request addresses. The uid and version
 // are server-assigned and unknown until the record loads, so they are omitted.

internal/ateattr/ateattr_test.go

@@ -135,3 +135,54 @@ func TestActorRefAttributes(t *testing.T) {
 		})
 	}
 }
+
+// TestKeySpellings pins the wire spelling of every key. Renaming one silently
+// breaks dashboards, alerts, and the contract between ateapi and atelet, so a
+// drift must fail here rather than in production.
+func TestKeySpellings(t *testing.T) {
+	tests := []struct {
+		key  attribute.Key
+		want string
+	}{
+		{AtespaceKey, "ate.atespace"},
+		{ActorNameKey, "ate.actor.name"},
+		{ActorUIDKey, "ate.actor.uid"},
+		{TemplateNameKey, "ate.template.name"},
+		{TemplateNamespaceKey, "ate.template.namespace"},
+		{ActorVersionKey, "ate.actor.version"},
+		{ActorOperationNameKey, "ate.actor.operation.name"},
+		{WorkerPoolNameKey, "ate.workerpool.name"},
+		{WorkerStateKey, "ate.worker.state"},
+		{SandboxClassKey, "ate.sandbox.class"},
+		{SnapshotKindKey, "ate.snapshot.kind"},
+		{SnapshotPhaseKey, "ate.snapshot.phase"},
+		{SchedulerOutcomeKey, "ate.scheduler.outcome"},
+		{ErrorTypeKey, "error.type"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.want, func(t *testing.T) {
+			if string(tt.key) != tt.want {
+				t.Errorf("key = %q, want %q", string(tt.key), tt.want)
+			}
+		})
+	}
+}
+
+func TestClampRestoreSnapshotKind(t *testing.T) {
+	tests := []struct {
+		in, want string
+	}{
+		{SnapshotKindGolden, SnapshotKindGolden},
+		{SnapshotKindLatest, SnapshotKindLatest},
+		{SnapshotKindBoot, SnapshotKindUnknown},
+		{"", SnapshotKindUnknown},
+		{"$(rm -rf)", SnapshotKindUnknown},
+	}
+	for _, tt := range tests {
+		t.Run(tt.in, func(t *testing.T) {
+			if got := ClampRestoreSnapshotKind(tt.in); got != tt.want {
+				t.Errorf("ClampRestoreSnapshotKind(%q) = %q, want %q", tt.in, got, tt.want)
+			}
+		})
+	}
+}