fix: recover from warm pool pod deletion instead of permanent error loop

Cherry-picks two upstream fixes:

1. kubernetes-sigs#521 — When an adopted warm pool pod is
   deleted (node failure, drain, eviction), the controller returned a hard
   error because the agents.x-k8s.io/pod-name annotation pointed to a
   non-existent pod. This left the Sandbox stuck in a permanent reconcile
   error loop. Now the controller clears the stale annotation and falls
   through to create a replacement pod (which remounts the existing PVC).

2. kubernetes-sigs#469 — During warm pool adoption, ensure
   the pod-name annotation is correct before the sandbox can be observed
   as Ready. Prevents stale annotations from being set in the first place.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: vamsi-resolve

controllers/sandbox_controller.go

@@ -477,8 +477,12 @@ func (r *SandboxReconciler) reconcilePod(ctx context.Context, sandbox *sandboxv1
 			return nil, fmt.Errorf("pod get failed: %w", err)
 		}
 		if podNameAnnotationExists {
-			log.Error(err, "Pod not found")
-			return nil, fmt.Errorf("pod in annotation get failed: %w", err)
+			log.Info("Tracked pod not found, clearing stale annotation", "podName", podName)
+			patch := client.MergeFrom(sandbox.DeepCopy())
+			delete(sandbox.Annotations, sandboxv1alpha1.SandboxPodNameAnnotation)
+			if patchErr := r.Patch(ctx, sandbox, patch); patchErr != nil {
+				return nil, fmt.Errorf("failed to clear stale pod name annotation: %w", patchErr)
+			}
 		}
 		pod = nil
 	}

controllers/sandbox_controller_test.go

@@ -1130,31 +1130,40 @@ func TestReconcilePod(t *testing.T) {
 			expectErr: true,
 		},
 		{
-			name:        "error when annotated pod does not exist",
-			initialObjs: []runtime.Object{},
-			sandbox: &sandboxv1alpha1.Sandbox{
+			name: "clears stale annotation and creates replacement pod when annotated pod does not exist",
+			sandbox: func() *sandboxv1alpha1.Sandbox {
+				s := sandboxObj.DeepCopy()
+				s.Annotations = map[string]string{
+					sandboxv1alpha1.SandboxPodNameAnnotation: "non-existent-pod",
+				}
+				return s
+			}(),
+			wantPod: &corev1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
-					Name:      sandboxName,
-					Namespace: sandboxNs,
+					Name:            sandboxName,
+					Namespace:       sandboxNs,
+					ResourceVersion: "1",
+					Labels: map[string]string{
+						"agents.x-k8s.io/sandbox-name-hash": nameHash,
+						"custom-label":                      "label-val",
+					},
 					Annotations: map[string]string{
-						sandboxv1alpha1.SandboxPodNameAnnotation: "non-existent-pod",
+						"custom-annotation": "anno-val",
 					},
+					OwnerReferences: []metav1.OwnerReference{sandboxControllerRef(sandboxName)},
 				},
-				Spec: sandboxv1alpha1.SandboxSpec{
-					Replicas: ptr.To(int32(1)),
-					PodTemplate: sandboxv1alpha1.PodTemplate{
-						Spec: corev1.PodSpec{
-							Containers: []corev1.Container{
-								{
-									Name: "test-container",
-								},
-							},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: "test-container",
 						},
 					},
 				},
 			},
-			wantPod:   nil,
-			expectErr: true,
+			expectErr: false,
+			wantSandboxAnnotations: map[string]string{
+				sandboxv1alpha1.SandboxPodNameAnnotation: sandboxName,
+			},
 		},
 		{
 			name: "refuses to delete annotated pod owned by a different controller",

extensions/controllers/sandboxclaim_controller.go

@@ -442,6 +442,13 @@ func (r *SandboxClaimReconciler) adoptSandboxFromCandidates(ctx context.Context,
 		if adopted.Annotations == nil {
 			adopted.Annotations = make(map[string]string)
 		}
+		// Ensure the adopted sandbox records its pod name before it can be observed Ready.
+		if podName := adopted.Annotations[v1alpha1.SandboxPodNameAnnotation]; podName != adopted.Name {
+			if podName != "" {
+				logger.Info("Correcting adopted sandbox pod-name annotation", "sandbox", adopted.Name, "oldPodName", podName, "newPodName", adopted.Name)
+			}
+			adopted.Annotations[v1alpha1.SandboxPodNameAnnotation] = adopted.Name
+		}
 		if tc, ok := claim.Annotations[asmetrics.TraceContextAnnotation]; ok {
 			adopted.Annotations[asmetrics.TraceContextAnnotation] = tc
 		}