fix(backend): guard ASR CUDA loads and reconciliation shutdown

CHANGELOG.md

@@ -22,6 +22,8 @@ metadata and the backend fallback mirror it.
 
 ### Fixed
 
+- Repeated backend lifespans wait for their own profile-reconciliation worker during shutdown (#2861) — thanks @rahul05ranjan!
+
 - Snap packages follow Canonical’s strict-confinement approach to launch without manual Chromium sandbox setup (#2893) — thanks @velixio!
 - Snap launches explain missing sandbox permissions instead of crashing before startup (#2889)
 - Snap packages expand the Electron desktop runtime so they launch after installation (#2858)

backend/main.py

@@ -915,24 +915,22 @@ async def _deferred_startup(app: FastAPI) -> None:
 # Same lifecycle markers as Phase A: cancelling the task cannot stop the thread
 # already importing engine runtimes, so shutdown waits (bounded) for it rather
 # than let interpreter teardown race a native import (#1000 class).
-_profile_reconcile_started = threading.Event()
-_profile_reconcile_finished = threading.Event()
-
-
-def _reconcile_profile_blocking():
-    _profile_reconcile_started.set()
+def _reconcile_profile_blocking(started: threading.Event, finished: threading.Event):
+    started.set()
     try:
         from services.performance_profiles import reconcile_active_profile
 
         return reconcile_active_profile()
     finally:
-        _profile_reconcile_finished.set()
+        finished.set()
 
 
-async def _reconcile_profile_in_background() -> None:
+async def _reconcile_profile_in_background(
+    started: threading.Event, finished: threading.Event,
+) -> None:
     """Reapply the saved performance profile off the startup critical path."""
     try:
-        recovered = await asyncio.to_thread(_reconcile_profile_blocking)
+        recovered = await asyncio.to_thread(_reconcile_profile_blocking, started, finished)
         if recovered:
             logger.info("Startup performance selections reconciled: %s", recovered)
     except asyncio.CancelledError:
@@ -941,6 +939,26 @@ async def _reconcile_profile_in_background() -> None:
         logger.exception("Performance-profile reconciliation failed (non-fatal).")
 
 
+def _start_profile_reconcile(app: FastAPI) -> None:
+    """Schedule reconciliation with markers owned by this lifespan."""
+    # Each worker captures its own markers: a prior lifespan completing late
+    # cannot mark this boot's worker finished (#2861).
+    events = (threading.Event(), threading.Event())
+    app.state.profile_reconcile_events = events
+    app.state.profile_reconcile_task = asyncio.create_task(
+        _reconcile_profile_in_background(*events)
+    )
+
+
+async def _wait_for_profile_reconcile_thread(app: FastAPI) -> None:
+    """Bound shutdown waiting on this boot's worker, even if its task was cancelled."""
+    events = getattr(app.state, "profile_reconcile_events", None)
+    if events is not None:
+        started, finished = events
+        if started.is_set() and not finished.is_set():
+            await asyncio.to_thread(finished.wait, 20.0)
+
+
 async def _await_profile_reconcile(app: FastAPI) -> None:
     """Let model warm-ups wait for reconciliation without inheriting its failure."""
     task = getattr(app.state, "profile_reconcile_task", None)
@@ -1021,7 +1039,7 @@ async def _phase_b(app: FastAPI) -> None:
     # on a cold disk), and nothing the first screen shows depends on the result,
     # so it runs in the background instead of holding /health at 503. The model
     # warm-ups below await it, so they still load the reconciled selections.
-    app.state.profile_reconcile_task = asyncio.create_task(_reconcile_profile_in_background())
+    _start_profile_reconcile(app)
     if _EAGER:
         # Eager boots (pytest, OMNIVOICE_EAGER_INIT=1) promise the classic
         # synchronous startup: callers read engines the moment the lifespan
@@ -1252,8 +1270,7 @@ async def lifespan(app: FastAPI):
     # before Phase A began pays nothing.
     if _phase_a_started.is_set() and not _phase_a_finished.is_set():
         await asyncio.to_thread(_phase_a_finished.wait, 20.0)
-    if _profile_reconcile_started.is_set() and not _profile_reconcile_finished.is_set():
-        await asyncio.to_thread(_profile_reconcile_finished.wait, 20.0)
+    await _wait_for_profile_reconcile_thread(app)
     try:
         from services.telephony import gateway as telephony_gateway
         await telephony_gateway.stop()

docs/electron-runtime.md

@@ -130,3 +130,8 @@ The VoiceStudio title links to `https://voicestudio.sh`. Hover or keyboard focus
 Website requests begin only after clicking. Remote pages use a sandboxed Electron WebContentsView with no preload, Node access, app bridge, or device permissions. Links and redirects stay restricted to HTTP(S); popup links reuse the same view. The browsing session is temporary and separate from app storage. The sidebar and workspace remain visible behind the modal. Its native page follows the modal bounds on resize and zoom, leaving the toolbar and surrounding workspace uncovered. **Back to Studio**, Escape, or clicking outside the modal disposes the embedded web contents, clears its session data, and restores the existing workspace and keyboard focus without navigating or remounting it. The browser can be reused for future cloud signup and other web flows; authentication callbacks are not implemented yet. Sites requiring device permissions or external authentication may need **Open in browser**. The browser list passes URLs as arguments to detected executables, never to a command shell.
 
 The Dubbing workspace keeps its existing `/dub` route and saved project format; the name change affects visible feature labels only.
+
+Backend embedders that restart the application lifespan in one Python process
+receive fresh performance-profile reconciliation markers on every boot. Shutdown
+waits up to 20 seconds for that boot's running reconciliation thread, including
+when its asyncio task was cancelled; a completed earlier boot cannot bypass the wait.

tests/test_startup_profile_reconcile_background.py

@@ -15,7 +15,7 @@
 
 def test_phase_b_does_not_reconcile_inline():
     assert "reconcile_active_profile" not in inspect.getsource(main._phase_b)
-    assert "_reconcile_profile_in_background" in inspect.getsource(main._phase_b)
+    assert "_start_profile_reconcile(app)" in inspect.getsource(main._phase_b)
 
 
 def test_warmups_wait_for_reconcile_in_background(monkeypatch):
@@ -31,9 +31,7 @@ def slow_reconcile():
 
     async def scenario():
         app = SimpleNamespace(state=SimpleNamespace())
-        app.state.profile_reconcile_task = asyncio.create_task(
-            main._reconcile_profile_in_background()
-        )
+        main._start_profile_reconcile(app)
         waiter = asyncio.create_task(main._await_profile_reconcile(app))
         await asyncio.sleep(0.05)
         # The caller (startup) got control back while reconcile is still running.
@@ -54,11 +52,10 @@ def broken():
 
     async def scenario():
         app = SimpleNamespace(state=SimpleNamespace())
-        app.state.profile_reconcile_task = asyncio.create_task(
-            main._reconcile_profile_in_background()
-        )
+        main._start_profile_reconcile(app)
         await main._await_profile_reconcile(app)
         assert app.state.profile_reconcile_task.exception() is None
+        assert app.state.profile_reconcile_events[1].is_set()
 
     asyncio.run(scenario())
 
@@ -105,4 +102,50 @@ def test_eager_boots_keep_the_classic_synchronous_startup():
 
 def test_shutdown_waits_for_a_running_reconcile_thread():
     source = inspect.getsource(main.lifespan)
-    assert "_profile_reconcile_finished.wait" in source
+    assert "_wait_for_profile_reconcile_thread(app)" in source
+
+
+def test_each_boot_waits_for_its_own_reconcile_after_task_cancellation(monkeypatch):
+    """A completed first boot must not suppress the second worker's shutdown wait."""
+    releases = [threading.Event(), threading.Event()]
+    boot = iter(range(2))
+
+    def reconcile():
+        releases[next(boot)].wait(5)
+        return {}
+
+    monkeypatch.setattr(importlib.import_module("services.performance_profiles"),
+                        "reconcile_active_profile", reconcile)
+
+    async def scenario():
+        app = SimpleNamespace(state=SimpleNamespace())
+        previous = None
+        for release in releases:
+            main._start_profile_reconcile(app)
+            started, finished = app.state.profile_reconcile_events
+            assert (started, finished) != previous
+            previous = (started, finished)
+            try:
+                assert await asyncio.to_thread(started.wait, 2)
+                task = app.state.profile_reconcile_task
+                task.cancel()
+                try:
+                    await task
+                except asyncio.CancelledError:
+                    pass
+                waiter = asyncio.create_task(main._wait_for_profile_reconcile_thread(app))
+                await asyncio.sleep(0.05)
+                assert not waiter.done()
+                assert not finished.is_set()
+                release.set()
+                await asyncio.wait_for(waiter, 2)
+                assert finished.is_set()
+            finally:
+                release.set()
+
+    asyncio.run(scenario())
+
+
+def test_shutdown_before_reconcile_scheduled_returns_immediately():
+    app = SimpleNamespace(state=SimpleNamespace())
+    asyncio.run(main._wait_for_profile_reconcile_thread(app))