fix(daemon): event-driven worktree reconcile check so cleanup self-heals

daemon/README.md

@@ -6,10 +6,13 @@ to SQLite, acknowledges new sessions asynchronously, and runs bloom-planner disc
 per-issue git worktrees. Planner turns stream Claude progress to Linear, persist terminal
 activities for retry, and resume the stored Claude session on follow-up prompts. Implementer
 assignments run a fresh, unattended literal `/do <identifier>` turn in the same issue worktree,
-durably attach an opened PR to the Linear session, and clean up clean worktrees after completed
-Issue webhooks. Follow-up replies to an implementer session resume its stored Claude session
+durably attach an opened PR to the Linear session, and clean up worktrees after Linear confirms
+their issues completed or canceled. Issue webhooks remain the fast path; an event-driven
+reconciliation sweep also runs at startup and after session completion, with no standing sweep
+timer. Follow-up replies to an implementer session resume its stored Claude session
 the same way planner prompts do, so a human can answer an implementer's question mid-stream.
-Dirty worktrees are retained and reported to the session.
+Dirty or PR-less state is pushed to its `agents/<id>` branch (or saved as a git bundle) before
+force removal.
 
 ## Local checks
 

daemon/ops/runbook.md

@@ -167,6 +167,8 @@ LINEAR_MCP_MONITOR_TIMEOUT_MS=10000
 SESSIONS_ENABLED=1
 TARGET_REPO_PATH=/var/lib/linear-agent-daemon/repos/bloom-mono
 WORKTREES_ROOT=/var/lib/linear-agent-daemon/worktrees
+WORKTREE_UNLINKED_GRACE_DAYS=14
+WORKTREE_BUNDLES_DIR=/var/lib/linear-agent-daemon/worktree-bundles
 LINEAR_API_KEY=...
 PLANNER_HARNESS=claude
 IMPLEMENTER_HARNESS=claude
@@ -793,7 +795,11 @@ elicitation/human-input request, `/do` starts on `agents/<identifier>`, a PR is
 the PR appears in session external URLs. Confirm the final `/do` text matches PR extraction.
 Move the issue to a workflow state whose stable type is `completed`; verify the Issues
 webhook arrives and both worktree and local branch disappear. Repeat with an uncommitted
-file and verify the worktree remains and a thought names its path. Confirm the full flow
+file and verify the worktree is removed only after a thought names the pushed
+`agents/<identifier>` branch or a bundle under `WORKTREE_BUNDLES_DIR`. Restore a pushed
+branch with `git fetch origin agents/<identifier>:agents/<identifier>`; inspect or restore
+a bundle with `git bundle verify <bundle>` followed by
+`git fetch <bundle> 'refs/heads/*:refs/remotes/bundle/*'`. Confirm the full flow
 under systemd hardening.
 
 ```bash

daemon/src/cleanup.ts

@@ -4,7 +4,7 @@ import type {
   CleanupNotificationRow,
 } from "./eventlog.js";
 import type { LinearGateway } from "./linear.js";
-import { WorktreeManager } from "./worktrees.js";
+import { WorktreeChangedError, WorktreeManager } from "./worktrees.js";
 import {
   buildInvocationSpan,
   buildSessionRoot,
@@ -27,6 +27,9 @@ export interface CleanupWorkerOptions {
   logger?: Logger;
   relay?: OtlpRelay;
   ingestDispatches?: () => Promise<void>;
+  worktrees?: WorktreeManager;
+  bundlesDir?: string;
+  onIssueFinalized?: () => void;
 }
 
 export class CleanupWorker {
@@ -39,13 +42,14 @@ export class CleanupWorker {
   constructor(
     private readonly log: EventLog,
     private readonly gateway: LinearGateway,
-    worktreesRoot: string,
+    private readonly worktreesRoot: string,
     targetRepoPath: string,
     private readonly options: CleanupWorkerOptions = {},
   ) {
     this.now = options.now ?? Date.now;
     this.logger = options.logger ?? console;
-    this.worktrees = new WorktreeManager(worktreesRoot, targetRepoPath);
+    this.worktrees =
+      options.worktrees ?? new WorktreeManager(worktreesRoot, targetRepoPath);
   }
   private readonly worktrees: WorktreeManager;
   start(): void {
@@ -82,6 +86,7 @@ export class CleanupWorker {
       await this.postNotification(note);
   }
   private async process(job: CleanupJobRow): Promise<void> {
+    let preservationAttempt = false;
     try {
       await this.options.ingestDispatches?.();
       if (!(await this.finalizeIssue(job))) {
@@ -92,36 +97,91 @@ export class CleanupWorker {
         );
         return;
       }
+      const expectedSnapshot = await this.worktrees.snapshot(
+        job.issueIdentifier,
+        { includeAbsent: true },
+      );
+      if (!expectedSnapshot)
+        throw new Error(
+          `Refusing cleanup without an owned worktree snapshot: ${job.issueIdentifier}`,
+        );
       const session = this.log.sessionByIssueIdentifier(job.issueIdentifier);
       if (
         !session?.worktreePath ||
         !(await this.worktrees.isPresent(session.worktreePath))
       ) {
-        await this.worktrees.remove(job.issueIdentifier);
+        preservationAttempt = true;
+        const result = await this.worktrees.preserveAndRemove(
+          job.issueIdentifier,
+          this.options.bundlesDir ?? `${this.worktreesRoot}/../worktree-bundles`,
+          { alwaysPreserve: true, expectedSnapshot },
+        );
+        preservationAttempt = false;
         this.log.clearSessionWorktrees(job.issueIdentifier);
+        this.log.noteCleanupOutcome(
+          job.id,
+          `Worktree cleanup completed; state ${result.detail}.`,
+          this.now(),
+        );
         this.log.markCleanupDone(job.id);
+        this.options.onIssueFinalized?.();
         return;
       }
       if (await this.worktrees.isClean(session.worktreePath)) {
         if (!this.log.hasExternalUrl(job.linearSessionId)) {
-          this.log.retainCleanup(
+          preservationAttempt = true;
+          const result = await this.worktrees.preserveAndRemove(
+            job.issueIdentifier,
+            this.options.bundlesDir ??
+              `${this.worktreesRoot}/../worktree-bundles`,
+            { alwaysPreserve: true, expectedSnapshot },
+          );
+          preservationAttempt = false;
+          this.log.clearSessionWorktrees(job.issueIdentifier);
+          this.log.noteCleanupOutcome(
             job.id,
-            `Worktree retained because no pull request was recorded; possible unpushed work is preserved: ${session.worktreePath}`,
+            `Worktree cleanup completed; state ${result.detail}.`,
             this.now(),
           );
+          this.log.markCleanupDone(job.id);
+          this.options.onIssueFinalized?.();
           return;
         }
-        await this.worktrees.remove(job.issueIdentifier);
+        await this.worktrees.remove(job.issueIdentifier, { expectedSnapshot });
         this.log.clearSessionWorktrees(job.issueIdentifier);
         this.log.markCleanupDone(job.id);
-      } else
-        this.log.retainCleanup(
+        this.options.onIssueFinalized?.();
+      } else {
+        preservationAttempt = true;
+        const result = await this.worktrees.preserveAndRemove(
+          job.issueIdentifier,
+          this.options.bundlesDir ?? `${this.worktreesRoot}/../worktree-bundles`,
+          { alwaysPreserve: true, expectedSnapshot },
+        );
+        preservationAttempt = false;
+        this.log.clearSessionWorktrees(job.issueIdentifier);
+        this.log.noteCleanupOutcome(
           job.id,
-          `Worktree retained because it is dirty: ${session.worktreePath}`,
+          `Worktree cleanup completed; state ${result.detail}.`,
           this.now(),
         );
+        this.log.markCleanupDone(job.id);
+        this.options.onIssueFinalized?.();
+      }
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
+      if (error instanceof WorktreeChangedError) {
+        this.log.retryCleanup(job.id, message, this.now() + 1000);
+        this.logger.log(
+          JSON.stringify({
+            event: "cleanup_revalidation_retry",
+            jobId: job.id,
+            issueIdentifier: job.issueIdentifier,
+            reason: "worktree_changed",
+          }),
+        );
+        return;
+      }
       if (
         this.now() <
         job.createdAt + (this.options.retryWindowMs ?? 30 * 60_000)
@@ -132,10 +192,18 @@ export class CleanupWorker {
           this.now() + Math.min(60_000, 1000 * 2 ** Math.min(job.attempts, 6)),
         );
       else {
-        this.log.failCleanup(job.id, message);
+        if (preservationAttempt)
+          this.log.retainCleanup(
+            job.id,
+            `Worktree retained because preservation failed: ${message}`,
+            this.now(),
+          );
+        else this.log.failCleanup(job.id, message);
         this.logger.error(
           JSON.stringify({
-            event: "cleanup_failed",
+            event: preservationAttempt
+              ? "cleanup_preservation_failed"
+              : "cleanup_failed",
             jobId: job.id,
             error: message,
           }),

daemon/src/config.ts

@@ -47,6 +47,8 @@ export interface Config {
   apps: Record<AppName, AppConfig>;
   sessionsEnabled: boolean;
   worktreesRoot: string;
+  worktreeUnlinkedGraceMs: number;
+  worktreeBundlesDir: string;
   targetRepoPath?: string;
   claudeArgv: string[];
   claudexArgv?: string[];
@@ -195,6 +197,15 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {
     apps: { planner: appConfig(env, "planner", testMode), implementer: appConfig(env, "implementer", testMode) },
     sessionsEnabled,
     worktreesRoot: env.WORKTREES_ROOT?.trim() || `${dirname(dbPath)}/worktrees`,
+    worktreeUnlinkedGraceMs:
+      positiveInteger(env, "WORKTREE_UNLINKED_GRACE_DAYS", 14) *
+      24 *
+      60 *
+      60 *
+      1000,
+    worktreeBundlesDir:
+      env.WORKTREE_BUNDLES_DIR?.trim() ||
+      `${dirname(dbPath)}/worktree-bundles`,
     ...(targetRepoPath ? { targetRepoPath } : {}),
     claudeArgv,
     ...(claudexArgv ? { claudexArgv } : {}),