diff --git a/controllers/sandbox_controller.go b/controllers/sandbox_controller.go index b4bc873f6..5bacf9645 100644 --- a/controllers/sandbox_controller.go +++ b/controllers/sandbox_controller.go @@ -477,8 +477,12 @@ func (r *SandboxReconciler) reconcilePod(ctx context.Context, sandbox *sandboxv1 return nil, fmt.Errorf("pod get failed: %w", err) } if podNameAnnotationExists { - log.Error(err, "Pod not found") - return nil, fmt.Errorf("pod in annotation get failed: %w", err) + log.Info("Tracked pod not found, clearing stale annotation", "podName", podName) + patch := client.MergeFrom(sandbox.DeepCopy()) + delete(sandbox.Annotations, sandboxv1alpha1.SandboxPodNameAnnotation) + if patchErr := r.Patch(ctx, sandbox, patch); patchErr != nil { + return nil, fmt.Errorf("failed to clear stale pod name annotation: %w", patchErr) + } } pod = nil } diff --git a/controllers/sandbox_controller_test.go b/controllers/sandbox_controller_test.go index c610b88d9..9f46df06e 100644 --- a/controllers/sandbox_controller_test.go +++ b/controllers/sandbox_controller_test.go @@ -1130,31 +1130,40 @@ func TestReconcilePod(t *testing.T) { expectErr: true, }, { - name: "error when annotated pod does not exist", - initialObjs: []runtime.Object{}, - sandbox: &sandboxv1alpha1.Sandbox{ + name: "clears stale annotation and creates replacement pod when annotated pod does not exist", + sandbox: func() *sandboxv1alpha1.Sandbox { + s := sandboxObj.DeepCopy() + s.Annotations = map[string]string{ + sandboxv1alpha1.SandboxPodNameAnnotation: "non-existent-pod", + } + return s + }(), + wantPod: &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: sandboxName, - Namespace: sandboxNs, + Name: sandboxName, + Namespace: sandboxNs, + ResourceVersion: "1", + Labels: map[string]string{ + "agents.x-k8s.io/sandbox-name-hash": nameHash, + "custom-label": "label-val", + }, Annotations: map[string]string{ - sandboxv1alpha1.SandboxPodNameAnnotation: "non-existent-pod", + "custom-annotation": "anno-val", }, + OwnerReferences: []metav1.OwnerReference{sandboxControllerRef(sandboxName)}, }, - Spec: sandboxv1alpha1.SandboxSpec{ - Replicas: ptr.To(int32(1)), - PodTemplate: sandboxv1alpha1.PodTemplate{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "test-container", - }, - }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test-container", }, }, }, }, - wantPod: nil, - expectErr: true, + expectErr: false, + wantSandboxAnnotations: map[string]string{ + sandboxv1alpha1.SandboxPodNameAnnotation: sandboxName, + }, }, { name: "refuses to delete annotated pod owned by a different controller", diff --git a/extensions/controllers/sandboxclaim_controller.go b/extensions/controllers/sandboxclaim_controller.go index 511538f15..8dbaa27e5 100644 --- a/extensions/controllers/sandboxclaim_controller.go +++ b/extensions/controllers/sandboxclaim_controller.go @@ -442,6 +442,13 @@ func (r *SandboxClaimReconciler) adoptSandboxFromCandidates(ctx context.Context, if adopted.Annotations == nil { adopted.Annotations = make(map[string]string) } + // Ensure the adopted sandbox records its pod name before it can be observed Ready. + if podName := adopted.Annotations[v1alpha1.SandboxPodNameAnnotation]; podName != adopted.Name { + if podName != "" { + logger.Info("Correcting adopted sandbox pod-name annotation", "sandbox", adopted.Name, "oldPodName", podName, "newPodName", adopted.Name) + } + adopted.Annotations[v1alpha1.SandboxPodNameAnnotation] = adopted.Name + } if tc, ok := claim.Annotations[asmetrics.TraceContextAnnotation]; ok { adopted.Annotations[asmetrics.TraceContextAnnotation] = tc }