Add more granularity around 1000ms in the Claim startup metrics

Author: igooch

dev/load-test/test-recipes/README-rapid-burst.md

@@ -44,12 +44,18 @@ Before running this test, ensure the following prerequisites are met:
         args:
           - --leader-elect=true
           - --extensions
+          - --enable-pprof-debug
+          - --enable-tracing
+          - --zap-log-level=debug
+          - --zap-encoder=json
           - --kube-api-qps=1000
           - --kube-api-burst=1000
           - --sandbox-concurrent-workers=1000
           - --sandbox-claim-concurrent-workers=1000
           - --sandbox-warm-pool-concurrent-workers=1000
     ```
+  - If you are using tracing, see [GKE OTLP Metrics](https://docs.cloud.google.com/stackdriver/docs/otlp-metrics/deploy-collector)
+    for how to deploy the collector.
   - Apply your modified manifests to your cluster to install the agent-sandbox controller.
     ```bash
     cd ~/agent-sandbox

dev/load-test/test-recipes/run_rapid_burst.sh

@@ -20,15 +20,15 @@ if [ -n "$1" ]; then
 fi
 
 # BURST_SIZE * TOTAL_BURSTS = Total sandbox claims created
-BURST_SIZE=1000
-QPS=1000
-TOTAL_BURSTS=10
-WARMPOOL_SIZE=1000
+BURST_SIZE=300
+QPS=300
+TOTAL_BURSTS=2
+WARMPOOL_SIZE=600
 RUNTIME_CLASS="" # Change to "gvisor" if your cluster supports it
 
 # Update these paths to match your environment
 # Clusterloader2 must be cloned or forked from https://github.com/kubernetes/perf-tests
-CL2_DIR="${HOME}/perf-tests/clusterloader2"
+CL2_DIR="${HOME}/oss/perf-tests/clusterloader2"
 AGENTS_DIR="${HOME}/agent-sandbox"
 TEST_DIR="${AGENTS_DIR}/dev/load-test/test-recipes"
 TEST_CONFIG="${TEST_DIR}/rapid-burst-test.yaml"

extensions/controllers/sandboxclaim_controller.go

@@ -833,7 +833,7 @@ func (r *SandboxClaimReconciler) recordCreationLatencyMetric(
 	asmetrics.RecordClaimStartupLatency(claim.CreationTimestamp.Time, launchType, claim.Spec.TemplateRef.Name)
 
 	// Record controller startup latency
-	if claim.Annotations != nil && claim.Annotations[observabilityAnnotation] != "" {
+	if claim.Annotations[observabilityAnnotation] != "" {
 		observedTimeString := claim.Annotations[observabilityAnnotation]
 		observedTime, err := time.Parse(time.RFC3339Nano, observedTimeString)
 		if err != nil {

internal/metrics/metrics.go

@@ -37,8 +37,8 @@ var (
 		prometheus.HistogramOpts{
 			Name: "agent_sandbox_claim_startup_latency_ms",
 			Help: "End-to-end latency from SandboxClaim creation to Sandbox Ready state in milliseconds.",
-			// Buckets for latency from 50ms to 4 minutes
-			Buckets: []float64{50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000, 240000},
+			// Buckets for latency from 100ms to 4 minutes
+			Buckets: []float64{100, 250, 500, 750, 1000, 1250, 1500, 2000, 2500, 5000, 10000, 30000, 60000, 120000, 240000},
 		},
 		[]string{"launch_type", "sandbox_template"},
 	)
@@ -51,8 +51,8 @@ var (
 		prometheus.HistogramOpts{
 			Name: "agent_sandbox_claim_controller_startup_latency_ms",
 			Help: "Latency from controller first observed SandboxClaim to Sandbox Ready state in milliseconds.",
-			// Buckets for latency from 50ms to 4 minutes
-			Buckets: []float64{50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000, 240000},
+			// Buckets for latency from 100ms to 4 minutes
+			Buckets: []float64{100, 250, 500, 750, 1000, 1250, 1500, 2000, 2500, 5000, 10000, 30000, 60000, 120000, 240000},
 		},
 		[]string{"launch_type", "sandbox_template"},
 	)