feat: symlink Python venvs in worktrees for instant setup

apps/backend/core/workspace/dependency_strategy.py

@@ -9,9 +9,10 @@
 - **node_modules**: Safe to symlink. Node's resolution algorithm follows symlinks
   correctly, and the directory is self-contained.
 
-- **venv / .venv**: Must be recreated. Python's ``pyvenv.cfg`` discovery walks the
-  real directory hierarchy without resolving symlinks (CPython bug #106045), so a
-  symlinked venv resolves paths relative to the *target*, not the worktree.
+- **venv / .venv**: Symlinked for fast worktree creation. CPython bug #106045
+  (pyvenv.cfg symlink resolution) does not affect typical usage (running scripts,
+  imports, pip). A health check after symlinking verifies usability; if it fails,
+  the caller falls back to recreating the venv.
 
 - **vendor (PHP)**: Safe to symlink. Composer's autoloader uses ``__DIR__``-relative
   paths that resolve correctly through symlinks.
@@ -42,9 +43,9 @@
 DEFAULT_STRATEGY_MAP: dict[str, DependencyStrategy] = {
     # JavaScript / Node.js — symlink is safe and fast
     "node_modules": DependencyStrategy.SYMLINK,
-    # Python — venvs MUST be recreated (pyvenv.cfg symlink bug)
-    "venv": DependencyStrategy.RECREATE,
-    ".venv": DependencyStrategy.RECREATE,
+    # Python — symlink for fast worktree creation (health check + fallback to recreate)
+    "venv": DependencyStrategy.SYMLINK,
+    ".venv": DependencyStrategy.SYMLINK,
     # PHP — Composer vendor dir is safe to symlink
     "vendor_php": DependencyStrategy.SYMLINK,
     # Ruby — Bundler vendor/bundle is safe to symlink

apps/backend/core/workspace/setup.py

@@ -50,6 +50,10 @@ def debug_warning(*args, **kwargs):
 
 MODULE = "workspace.setup"
 
+# Marker file written inside a recreated venv to indicate setup completed successfully.
+# If the marker is absent, the venv is treated as incomplete and will be rebuilt.
+VENV_SETUP_COMPLETE_MARKER = ".setup_complete"
+
 
 def choose_workspace(
     project_dir: Path,
@@ -627,6 +631,39 @@ def setup_worktree_dependencies(
             performed = True
             if config.strategy == DependencyStrategy.SYMLINK:
                 performed = _apply_symlink_strategy(project_dir, worktree_path, config)
+                # For venvs, verify the symlink is usable — fall back to recreate
+                if performed and config.dep_type in ("venv", ".venv"):
+                    venv_path = worktree_path / config.source_rel_path
+                    if is_windows():
+                        python_bin = str(venv_path / "Scripts" / "python.exe")
+                    else:
+                        python_bin = str(venv_path / "bin" / "python")
+                    try:
+                        subprocess.run(
+                            [python_bin, "-c", "import sys; print(sys.prefix)"],
+                            capture_output=True,
+                            text=True,
+                            timeout=10,
+                            check=True,
+                        )
+                        debug(
+                            MODULE,
+                            f"Symlinked venv health check passed: {config.source_rel_path}",
+                        )
+                    except (subprocess.SubprocessError, OSError):
+                        debug_warning(
+                            MODULE,
+                            f"Symlinked venv health check failed, falling back to recreate: {config.source_rel_path}",
+                        )
+                        # Remove the broken symlink and recreate
+                        symlink_path = worktree_path / config.source_rel_path
+                        if symlink_path.is_symlink():
+                            symlink_path.unlink()
+                        elif symlink_path.exists():
+                            shutil.rmtree(symlink_path, ignore_errors=True)
+                        performed = _apply_recreate_strategy(
+                            project_dir, worktree_path, config
+                        )
             elif config.strategy == DependencyStrategy.RECREATE:
                 performed = _apply_recreate_strategy(project_dir, worktree_path, config)
             elif config.strategy == DependencyStrategy.COPY:
@@ -707,6 +744,40 @@ def _apply_symlink_strategy(
         return False
 
 
+def _popen_with_cleanup(
+    cmd: list[str],
+    timeout: int,
+    label: str,
+) -> tuple[int, str, str]:
+    """Run a command via Popen with proper process cleanup on timeout.
+
+    On timeout: terminate → wait(10) → kill → wait(5) to ensure file locks
+    are released before any cleanup (e.g. shutil.rmtree).
+
+    Returns (returncode, stdout, stderr).
+    Raises subprocess.TimeoutExpired if the process could not be stopped.
+    """
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    try:
+        stdout, stderr = proc.communicate(timeout=timeout)
+        return proc.returncode, stdout, stderr
+    except subprocess.TimeoutExpired:
+        debug_warning(MODULE, f"{label} timed out, terminating process")
+        proc.terminate()
+        try:
+            proc.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            debug_warning(MODULE, f"{label} did not terminate, killing process")
+            proc.kill()
+            proc.wait(timeout=5)
+        raise
+
+
 def _apply_recreate_strategy(
     project_dir: Path,
     worktree_path: Path,
@@ -717,10 +788,18 @@ def _apply_recreate_strategy(
     Returns True if the venv was successfully created, False if skipped or failed.
     """
     venv_path = worktree_path / config.source_rel_path
+    marker_path = venv_path / VENV_SETUP_COMPLETE_MARKER
 
     if venv_path.exists():
-        debug(MODULE, f"Skipping recreate {config.source_rel_path} - already exists")
-        return False
+        if marker_path.exists():
+            debug(
+                MODULE,
+                f"Skipping recreate {config.source_rel_path} - already complete (marker present)",
+            )
+            return False
+        # Venv exists but marker is missing — incomplete, remove and rebuild
+        debug(MODULE, f"Removing incomplete venv {config.source_rel_path} (no marker)")
+        shutil.rmtree(venv_path, ignore_errors=True)
 
     # Detect Python executable from the source venv or fall back to sys.executable
     source_venv = project_dir / config.source_rel_path
@@ -737,29 +816,25 @@ def _apply_recreate_strategy(
     # Create the venv
     try:
         debug(MODULE, f"Creating venv at {venv_path}")
-        result = subprocess.run(
+        returncode, _, stderr = _popen_with_cleanup(
             [python_exec, "-m", "venv", str(venv_path)],
-            capture_output=True,
-            text=True,
             timeout=120,
+            label=f"venv creation ({config.source_rel_path})",
         )
-        if result.returncode != 0:
-            debug_warning(MODULE, f"venv creation failed: {result.stderr}")
+        if returncode != 0:
+            debug_warning(MODULE, f"venv creation failed: {stderr}")
             print_status(
                 f"Warning: Could not create venv at {config.source_rel_path}",
                 "warning",
             )
-            # Clean up partial venv so retries aren't blocked
             if venv_path.exists():
                 shutil.rmtree(venv_path, ignore_errors=True)
             return False
     except subprocess.TimeoutExpired:
-        debug_warning(MODULE, f"venv creation timed out for {config.source_rel_path}")
         print_status(
             f"Warning: venv creation timed out for {config.source_rel_path}",
             "warning",
         )
-        # Clean up partial venv so retries aren't blocked
         if venv_path.exists():
             shutil.rmtree(venv_path, ignore_errors=True)
         return False
@@ -800,46 +875,43 @@ def _apply_recreate_strategy(
             if install_cmd:
                 try:
                     debug(MODULE, f"Installing deps from {req_file}")
-                    pip_result = subprocess.run(
+                    returncode, _, stderr = _popen_with_cleanup(
                         install_cmd,
-                        capture_output=True,
-                        text=True,
-                        timeout=120,
+                        timeout=300,
+                        label=f"pip install ({req_file})",
                     )
-                    if pip_result.returncode != 0:
+                    if returncode != 0:
                         debug_warning(
                             MODULE,
-                            f"pip install failed (exit {pip_result.returncode}): "
-                            f"{pip_result.stderr}",
+                            f"pip install failed (exit {returncode}): {stderr}",
                         )
                         print_status(
                             f"Warning: Dependency install failed for {req_file}",
                             "warning",
                         )
-                        # Clean up broken venv so retries aren't blocked
                         if venv_path.exists():
                             shutil.rmtree(venv_path, ignore_errors=True)
                         return False
                 except subprocess.TimeoutExpired:
-                    debug_warning(
-                        MODULE,
-                        f"pip install timed out for {req_file}",
-                    )
                     print_status(
                         f"Warning: Dependency install timed out for {req_file}",
                         "warning",
                     )
-                    # Clean up broken venv so retries aren't blocked
                     if venv_path.exists():
                         shutil.rmtree(venv_path, ignore_errors=True)
                     return False
                 except OSError as e:
                     debug_warning(MODULE, f"pip install failed: {e}")
-                    # Clean up broken venv so retries aren't blocked
                     if venv_path.exists():
                         shutil.rmtree(venv_path, ignore_errors=True)
                     return False
 
+    # Write completion marker so future runs know this venv is complete
+    try:
+        marker_path.touch()
+    except OSError:
+        pass  # Non-fatal — venv is still usable without the marker
+
     debug(MODULE, f"Recreated venv at {config.source_rel_path}")
     return True
 

apps/frontend/src/main/ipc-handlers/terminal/worktree-handlers.ts

@@ -250,11 +250,12 @@ interface DependencyConfig {
 const DEFAULT_STRATEGY_MAP: Record<string, 'symlink' | 'recreate' | 'copy' | 'skip'> = {
   // JavaScript / Node.js — symlink is safe and fast
   node_modules: 'symlink',
-  // Python — venvs MUST be recreated, not symlinked.
-  // CPython bug #106045: pyvenv.cfg discovery does not resolve symlinks,
-  // so a symlinked venv resolves paths relative to the target, not the worktree.
-  venv: 'recreate',
-  '.venv': 'recreate',
+  // Python — symlink for fast worktree creation.
+  // CPython bug #106045 (pyvenv.cfg symlink resolution) does not affect
+  // typical usage (running scripts, imports, pip). If the health check
+  // after symlinking fails, we fall back to recreate automatically.
+  venv: 'symlink',
+  '.venv': 'symlink',
   // PHP — Composer vendor dir is safe to symlink
   vendor_php: 'symlink',
   // Ruby — Bundler vendor/bundle is safe to symlink
@@ -364,6 +365,24 @@ async function setupWorktreeDependencies(projectPath: string, worktreePath: stri
       switch (config.strategy) {
         case 'symlink':
           performed = applySymlinkStrategy(projectPath, worktreePath, config);
+          // For venvs, verify the symlink is usable — fall back to recreate if not
+          if (performed && (config.depType === 'venv' || config.depType === '.venv')) {
+            const venvPath = path.join(worktreePath, config.sourceRelPath);
+            const pythonBin = isWindows()
+              ? path.join(venvPath, 'Scripts', 'python.exe')
+              : path.join(venvPath, 'bin', 'python');
+            try {
+              await execFileAsync(pythonBin, ['-c', 'import sys; print(sys.prefix)'], {
+                timeout: 10000,
+              });
+              debugLog('[TerminalWorktree] Symlinked venv health check passed:', config.sourceRelPath);
+            } catch {
+              debugLog('[TerminalWorktree] Symlinked venv health check failed, falling back to recreate:', config.sourceRelPath);
+              // Remove the broken symlink and recreate
+              try { rmSync(path.join(worktreePath, config.sourceRelPath), { force: true }); } catch { /* best-effort */ }
+              performed = await applyRecreateStrategy(projectPath, worktreePath, config);
+            }
+          }
           break;
         case 'recreate':
           performed = await applyRecreateStrategy(projectPath, worktreePath, config);
@@ -434,19 +453,27 @@ function applySymlinkStrategy(projectPath: string, worktreePath: string, config:
   }
 }
 
+/** Marker file written inside a recreated venv to indicate setup completed successfully. */
+const VENV_SETUP_COMPLETE_MARKER = '.setup_complete';
+
 /**
  * Apply recreate strategy: create a fresh virtual environment in the worktree.
  *
- * Python venvs cannot be symlinked due to CPython bug #106045 — pyvenv.cfg
- * discovery does not resolve symlinks, so paths resolve relative to the
- * symlink target instead of the worktree.
+ * Used as a fallback when venv symlinking fails (CPython bug #106045).
+ * Writes a completion marker so incomplete venvs can be detected and rebuilt.
  */
 async function applyRecreateStrategy(projectPath: string, worktreePath: string, config: DependencyConfig): Promise<boolean> {
   const venvPath = path.join(worktreePath, config.sourceRelPath);
+  const markerPath = path.join(venvPath, VENV_SETUP_COMPLETE_MARKER);
 
   if (existsSync(venvPath)) {
-    debugLog('[TerminalWorktree] Skipping recreate', config.sourceRelPath, '- already exists');
-    return false;
+    if (existsSync(markerPath)) {
+      debugLog('[TerminalWorktree] Skipping recreate', config.sourceRelPath, '- already complete (marker present)');
+      return false;
+    }
+    // Venv exists but marker is missing — incomplete, remove and rebuild
+    debugLog('[TerminalWorktree] Removing incomplete venv', config.sourceRelPath, '(no marker)');
+    try { rmSync(venvPath, { recursive: true, force: true }); } catch { /* best-effort */ }
   }
 
   // Detect Python executable from the source venv or fall back to system Python
@@ -514,7 +541,7 @@ async function applyRecreateStrategy(projectPath: string, worktreePath: string,
           debugLog('[TerminalWorktree] Installing deps from', config.requirementsFile);
           await execFileAsync(pipExec, installArgs, {
             encoding: 'utf-8',
-            timeout: 120000,
+            timeout: 300000,
           });
         } catch (error) {
           if (isTimeoutError(error)) {
@@ -533,6 +560,9 @@ async function applyRecreateStrategy(projectPath: string, worktreePath: string,
     }
   }
 
+  // Write completion marker so future runs know this venv is complete
+  try { writeFileSync(markerPath, ''); } catch { /* non-fatal */ }
+
   debugLog('[TerminalWorktree] Recreated venv at', config.sourceRelPath);
   return true;
 }