From 3ea988ac83ccaf40c5452bccd4642aff8931af7a Mon Sep 17 00:00:00 2001 From: Shimul Debnath <270455167+velixio@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:39:53 +0530 Subject: [PATCH 1/6] fix(startup): scope reconciliation shutdown markers to each lifespan --- CHANGELOG.md | 2 + backend/main.py | 41 +++++++++---- docs/electron-runtime.md | 5 ++ ...st_startup_profile_reconcile_background.py | 59 ++++++++++++++++--- 4 files changed, 87 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ab41f333..230dea2e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ metadata and the backend fallback mirror it. ### Fixed +- Repeated backend lifespans wait for their own profile-reconciliation worker during shutdown (#2861) — thanks @rahul05ranjan! + - Snap packages follow Canonical’s strict-confinement approach to launch without manual Chromium sandbox setup (#2893) — thanks @velixio! - Snap launches explain missing sandbox permissions instead of crashing before startup (#2889) - Snap packages expand the Electron desktop runtime so they launch after installation (#2858) diff --git a/backend/main.py b/backend/main.py index d22e8769f..5f1aef550 100644 --- a/backend/main.py +++ b/backend/main.py @@ -915,24 +915,22 @@ async def _deferred_startup(app: FastAPI) -> None: # Same lifecycle markers as Phase A: cancelling the task cannot stop the thread # already importing engine runtimes, so shutdown waits (bounded) for it rather # than let interpreter teardown race a native import (#1000 class). -_profile_reconcile_started = threading.Event() -_profile_reconcile_finished = threading.Event() - - -def _reconcile_profile_blocking(): - _profile_reconcile_started.set() +def _reconcile_profile_blocking(started: threading.Event, finished: threading.Event): + started.set() try: from services.performance_profiles import reconcile_active_profile return reconcile_active_profile() finally: - _profile_reconcile_finished.set() + finished.set() -async def _reconcile_profile_in_background() -> None: +async def _reconcile_profile_in_background( + started: threading.Event, finished: threading.Event, +) -> None: """Reapply the saved performance profile off the startup critical path.""" try: - recovered = await asyncio.to_thread(_reconcile_profile_blocking) + recovered = await asyncio.to_thread(_reconcile_profile_blocking, started, finished) if recovered: logger.info("Startup performance selections reconciled: %s", recovered) except asyncio.CancelledError: @@ -941,6 +939,26 @@ async def _reconcile_profile_in_background() -> None: logger.exception("Performance-profile reconciliation failed (non-fatal).") +def _start_profile_reconcile(app: FastAPI) -> None: + """Schedule reconciliation with markers owned by this lifespan.""" + # Each worker captures its own markers: a prior lifespan completing late + # cannot mark this boot's worker finished (#2861). + events = (threading.Event(), threading.Event()) + app.state.profile_reconcile_events = events + app.state.profile_reconcile_task = asyncio.create_task( + _reconcile_profile_in_background(*events) + ) + + +async def _wait_for_profile_reconcile_thread(app: FastAPI) -> None: + """Bound shutdown waiting on this boot's worker, even if its task was cancelled.""" + events = getattr(app.state, "profile_reconcile_events", None) + if events is not None: + started, finished = events + if started.is_set() and not finished.is_set(): + await asyncio.to_thread(finished.wait, 20.0) + + async def _await_profile_reconcile(app: FastAPI) -> None: """Let model warm-ups wait for reconciliation without inheriting its failure.""" task = getattr(app.state, "profile_reconcile_task", None) @@ -1021,7 +1039,7 @@ async def _phase_b(app: FastAPI) -> None: # on a cold disk), and nothing the first screen shows depends on the result, # so it runs in the background instead of holding /health at 503. The model # warm-ups below await it, so they still load the reconciled selections. - app.state.profile_reconcile_task = asyncio.create_task(_reconcile_profile_in_background()) + _start_profile_reconcile(app) if _EAGER: # Eager boots (pytest, OMNIVOICE_EAGER_INIT=1) promise the classic # synchronous startup: callers read engines the moment the lifespan @@ -1252,8 +1270,7 @@ async def lifespan(app: FastAPI): # before Phase A began pays nothing. if _phase_a_started.is_set() and not _phase_a_finished.is_set(): await asyncio.to_thread(_phase_a_finished.wait, 20.0) - if _profile_reconcile_started.is_set() and not _profile_reconcile_finished.is_set(): - await asyncio.to_thread(_profile_reconcile_finished.wait, 20.0) + await _wait_for_profile_reconcile_thread(app) try: from services.telephony import gateway as telephony_gateway await telephony_gateway.stop() diff --git a/docs/electron-runtime.md b/docs/electron-runtime.md index 912d6fa49..9f897edd8 100644 --- a/docs/electron-runtime.md +++ b/docs/electron-runtime.md @@ -130,3 +130,8 @@ The VoiceStudio title links to `https://voicestudio.sh`. Hover or keyboard focus Website requests begin only after clicking. Remote pages use a sandboxed Electron WebContentsView with no preload, Node access, app bridge, or device permissions. Links and redirects stay restricted to HTTP(S); popup links reuse the same view. The browsing session is temporary and separate from app storage. The sidebar and workspace remain visible behind the modal. Its native page follows the modal bounds on resize and zoom, leaving the toolbar and surrounding workspace uncovered. **Back to Studio**, Escape, or clicking outside the modal disposes the embedded web contents, clears its session data, and restores the existing workspace and keyboard focus without navigating or remounting it. The browser can be reused for future cloud signup and other web flows; authentication callbacks are not implemented yet. Sites requiring device permissions or external authentication may need **Open in browser**. The browser list passes URLs as arguments to detected executables, never to a command shell. The Dubbing workspace keeps its existing `/dub` route and saved project format; the name change affects visible feature labels only. + +Backend embedders that restart the application lifespan in one Python process +receive fresh performance-profile reconciliation markers on every boot. Shutdown +waits up to 20 seconds for that boot's running reconciliation thread, including +when its asyncio task was cancelled; a completed earlier boot cannot bypass the wait. diff --git a/tests/test_startup_profile_reconcile_background.py b/tests/test_startup_profile_reconcile_background.py index 6c1e3f201..9545a7b9d 100644 --- a/tests/test_startup_profile_reconcile_background.py +++ b/tests/test_startup_profile_reconcile_background.py @@ -15,7 +15,7 @@ def test_phase_b_does_not_reconcile_inline(): assert "reconcile_active_profile" not in inspect.getsource(main._phase_b) - assert "_reconcile_profile_in_background" in inspect.getsource(main._phase_b) + assert "_start_profile_reconcile(app)" in inspect.getsource(main._phase_b) def test_warmups_wait_for_reconcile_in_background(monkeypatch): @@ -31,9 +31,7 @@ def slow_reconcile(): async def scenario(): app = SimpleNamespace(state=SimpleNamespace()) - app.state.profile_reconcile_task = asyncio.create_task( - main._reconcile_profile_in_background() - ) + main._start_profile_reconcile(app) waiter = asyncio.create_task(main._await_profile_reconcile(app)) await asyncio.sleep(0.05) # The caller (startup) got control back while reconcile is still running. @@ -54,11 +52,10 @@ def broken(): async def scenario(): app = SimpleNamespace(state=SimpleNamespace()) - app.state.profile_reconcile_task = asyncio.create_task( - main._reconcile_profile_in_background() - ) + main._start_profile_reconcile(app) await main._await_profile_reconcile(app) assert app.state.profile_reconcile_task.exception() is None + assert app.state.profile_reconcile_events[1].is_set() asyncio.run(scenario()) @@ -105,4 +102,50 @@ def test_eager_boots_keep_the_classic_synchronous_startup(): def test_shutdown_waits_for_a_running_reconcile_thread(): source = inspect.getsource(main.lifespan) - assert "_profile_reconcile_finished.wait" in source + assert "_wait_for_profile_reconcile_thread(app)" in source + + +def test_each_boot_waits_for_its_own_reconcile_after_task_cancellation(monkeypatch): + """A completed first boot must not suppress the second worker's shutdown wait.""" + releases = [threading.Event(), threading.Event()] + boot = iter(range(2)) + + def reconcile(): + releases[next(boot)].wait(5) + return {} + + monkeypatch.setattr(importlib.import_module("services.performance_profiles"), + "reconcile_active_profile", reconcile) + + async def scenario(): + app = SimpleNamespace(state=SimpleNamespace()) + previous = None + for release in releases: + main._start_profile_reconcile(app) + started, finished = app.state.profile_reconcile_events + assert (started, finished) != previous + previous = (started, finished) + try: + assert await asyncio.to_thread(started.wait, 2) + task = app.state.profile_reconcile_task + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + waiter = asyncio.create_task(main._wait_for_profile_reconcile_thread(app)) + await asyncio.sleep(0.05) + assert not waiter.done() + assert not finished.is_set() + release.set() + await asyncio.wait_for(waiter, 2) + assert finished.is_set() + finally: + release.set() + + asyncio.run(scenario()) + + +def test_shutdown_before_reconcile_scheduled_returns_immediately(): + app = SimpleNamespace(state=SimpleNamespace()) + asyncio.run(main._wait_for_profile_reconcile_thread(app)) From 71c97549d4fb18927a3471500a85e6edfdf5f29a Mon Sep 17 00:00:00 2001 From: Shimul Debnath <270455167+velixio@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:47:45 +0530 Subject: [PATCH 2/6] fix(startup): track queued reconciliation work through cancellation --- backend/main.py | 51 ++++++------- docs/electron-runtime.md | 4 +- ...st_startup_profile_reconcile_background.py | 73 ++++++++++++++++--- 3 files changed, 90 insertions(+), 38 deletions(-) diff --git a/backend/main.py b/backend/main.py index 5f1aef550..4e2596b02 100644 --- a/backend/main.py +++ b/backend/main.py @@ -912,25 +912,21 @@ async def _deferred_startup(app: FastAPI) -> None: os._exit(1) -# Same lifecycle markers as Phase A: cancelling the task cannot stop the thread -# already importing engine runtimes, so shutdown waits (bounded) for it rather -# than let interpreter teardown race a native import (#1000 class). -def _reconcile_profile_blocking(started: threading.Event, finished: threading.Event): - started.set() - try: - from services.performance_profiles import reconcile_active_profile +def _reconcile_profile_blocking(): + from services.performance_profiles import reconcile_active_profile - return reconcile_active_profile() - finally: - finished.set() + return reconcile_active_profile() -async def _reconcile_profile_in_background( - started: threading.Event, finished: threading.Event, -) -> None: +async def _reconcile_profile_in_background(app: FastAPI) -> None: """Reapply the saved performance profile off the startup critical path.""" try: - recovered = await asyncio.to_thread(_reconcile_profile_blocking, started, finished) + # Store only successfully submitted work. Shielding keeps cancellation + # of the asyncio wrapper from cancelling queued executor work, so the + # shutdown wait can track its actual completion (#2861). + worker = asyncio.get_running_loop().run_in_executor(None, _reconcile_profile_blocking) + app.state.profile_reconcile_worker = worker + recovered = await asyncio.shield(worker) if recovered: logger.info("Startup performance selections reconciled: %s", recovered) except asyncio.CancelledError: @@ -940,23 +936,22 @@ async def _reconcile_profile_in_background( def _start_profile_reconcile(app: FastAPI) -> None: - """Schedule reconciliation with markers owned by this lifespan.""" - # Each worker captures its own markers: a prior lifespan completing late - # cannot mark this boot's worker finished (#2861). - events = (threading.Event(), threading.Event()) - app.state.profile_reconcile_events = events - app.state.profile_reconcile_task = asyncio.create_task( - _reconcile_profile_in_background(*events) - ) + """Schedule reconciliation with a worker handle owned by this lifespan.""" + app.state.profile_reconcile_worker = None + app.state.profile_reconcile_task = asyncio.create_task(_reconcile_profile_in_background(app)) async def _wait_for_profile_reconcile_thread(app: FastAPI) -> None: - """Bound shutdown waiting on this boot's worker, even if its task was cancelled.""" - events = getattr(app.state, "profile_reconcile_events", None) - if events is not None: - started, finished = events - if started.is_set() and not finished.is_set(): - await asyncio.to_thread(finished.wait, 20.0) + """Stop new submissions and bound waiting for this boot's executor work.""" + task = getattr(app.state, "profile_reconcile_task", None) + if task is not None and not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + worker = getattr(app.state, "profile_reconcile_worker", None) + if worker is not None: + # asyncio.wait neither cancels the executor future on timeout nor + # needs a free executor thread to enforce the shutdown deadline. + await asyncio.wait({worker}, timeout=20.0) async def _await_profile_reconcile(app: FastAPI) -> None: diff --git a/docs/electron-runtime.md b/docs/electron-runtime.md index 9f897edd8..86668a11e 100644 --- a/docs/electron-runtime.md +++ b/docs/electron-runtime.md @@ -132,6 +132,8 @@ Website requests begin only after clicking. Remote pages use a sandboxed Electro The Dubbing workspace keeps its existing `/dub` route and saved project format; the name change affects visible feature labels only. Backend embedders that restart the application lifespan in one Python process -receive fresh performance-profile reconciliation markers on every boot. Shutdown +receive a fresh performance-profile reconciliation worker handle on every boot. Shutdown waits up to 20 seconds for that boot's running reconciliation thread, including when its asyncio task was cancelled; a completed earlier boot cannot bypass the wait. +Work not yet submitted is cancelled, and queued executor work remains tracked +without requiring another executor thread to enforce the shutdown deadline. diff --git a/tests/test_startup_profile_reconcile_background.py b/tests/test_startup_profile_reconcile_background.py index 9545a7b9d..994278f9e 100644 --- a/tests/test_startup_profile_reconcile_background.py +++ b/tests/test_startup_profile_reconcile_background.py @@ -55,7 +55,7 @@ async def scenario(): main._start_profile_reconcile(app) await main._await_profile_reconcile(app) assert app.state.profile_reconcile_task.exception() is None - assert app.state.profile_reconcile_events[1].is_set() + assert app.state.profile_reconcile_worker.done() asyncio.run(scenario()) @@ -108,10 +108,13 @@ def test_shutdown_waits_for_a_running_reconcile_thread(): def test_each_boot_waits_for_its_own_reconcile_after_task_cancellation(monkeypatch): """A completed first boot must not suppress the second worker's shutdown wait.""" releases = [threading.Event(), threading.Event()] + entered = [threading.Event(), threading.Event()] boot = iter(range(2)) def reconcile(): - releases[next(boot)].wait(5) + index = next(boot) + entered[index].set() + releases[index].wait(5) return {} monkeypatch.setattr(importlib.import_module("services.performance_profiles"), @@ -120,13 +123,13 @@ def reconcile(): async def scenario(): app = SimpleNamespace(state=SimpleNamespace()) previous = None - for release in releases: + for index, release in enumerate(releases): main._start_profile_reconcile(app) - started, finished = app.state.profile_reconcile_events - assert (started, finished) != previous - previous = (started, finished) try: - assert await asyncio.to_thread(started.wait, 2) + assert await asyncio.to_thread(entered[index].wait, 2) + worker = app.state.profile_reconcile_worker + assert worker is not previous + previous = worker task = app.state.profile_reconcile_task task.cancel() try: @@ -136,10 +139,11 @@ async def scenario(): waiter = asyncio.create_task(main._wait_for_profile_reconcile_thread(app)) await asyncio.sleep(0.05) assert not waiter.done() - assert not finished.is_set() + assert not worker.done() release.set() await asyncio.wait_for(waiter, 2) - assert finished.is_set() + assert worker.done() + assert not worker.cancelled() finally: release.set() @@ -149,3 +153,54 @@ async def scenario(): def test_shutdown_before_reconcile_scheduled_returns_immediately(): app = SimpleNamespace(state=SimpleNamespace()) asyncio.run(main._wait_for_profile_reconcile_thread(app)) + + +def test_shutdown_cancels_unsubmitted_reconcile(monkeypatch): + calls = [] + monkeypatch.setattr(importlib.import_module("services.performance_profiles"), + "reconcile_active_profile", lambda: calls.append(True)) + + async def scenario(): + app = SimpleNamespace(state=SimpleNamespace()) + main._start_profile_reconcile(app) + # No yield: the wrapper has not submitted executor work yet. + await main._wait_for_profile_reconcile_thread(app) + assert app.state.profile_reconcile_worker is None + assert app.state.profile_reconcile_task.cancelled() + assert calls == [] + + asyncio.run(scenario()) + + +def test_shutdown_waits_for_queued_reconcile_without_cancelling_it(monkeypatch): + from concurrent.futures import ThreadPoolExecutor + + release_executor = threading.Event() + calls = [] + monkeypatch.setattr(importlib.import_module("services.performance_profiles"), + "reconcile_active_profile", lambda: calls.append(True)) + + async def scenario(): + loop = asyncio.get_running_loop() + loop.set_default_executor(ThreadPoolExecutor(max_workers=1)) + blocker = loop.run_in_executor(None, release_executor.wait, 5) + app = SimpleNamespace(state=SimpleNamespace()) + try: + main._start_profile_reconcile(app) + await asyncio.sleep(0) # submitted behind the occupied executor + worker = app.state.profile_reconcile_worker + assert worker is not None + waiter = asyncio.create_task(main._wait_for_profile_reconcile_thread(app)) + await asyncio.sleep(0.05) + assert not waiter.done() + assert not worker.cancelled() + assert calls == [] + release_executor.set() + await asyncio.wait_for(waiter, 2) + await blocker + assert calls == [True] + assert worker.done() + finally: + release_executor.set() + + asyncio.run(scenario()) From a89e28beb1bf3630591cc3708c937f9dec00c707 Mon Sep 17 00:00:00 2001 From: Shimul Debnath <270455167+velixio@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:52:02 +0530 Subject: [PATCH 3/6] chore: resolve changelog merge to match main --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5ca501c1..7535c93c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,10 +22,7 @@ metadata and the backend fallback mirror it. ### Fixed -<<<<<<< HEAD -======= - Tools → Convert includes saved Voice Design profiles and clones with locked reference audio in the target voice picker (#2899) ->>>>>>> origin/main - Snap packages follow Canonical’s strict-confinement approach to launch without manual Chromium sandbox setup (#2893) — thanks @velixio! - Snap launches explain missing sandbox permissions instead of crashing before startup (#2889) From 4ff2dcc1b1fb2cb289a56372e8987afcf70cebd6 Mon Sep 17 00:00:00 2001 From: Shimul Debnath <270455167+velixio@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:54:57 +0530 Subject: [PATCH 4/6] fix(startup): cancel warmups when reconciliation is stopped --- backend/main.py | 2 ++ ...st_startup_profile_reconcile_background.py | 30 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/backend/main.py b/backend/main.py index 4e2596b02..2b26f9770 100644 --- a/backend/main.py +++ b/backend/main.py @@ -959,6 +959,8 @@ async def _await_profile_reconcile(app: FastAPI) -> None: task = getattr(app.state, "profile_reconcile_task", None) if task is not None: await asyncio.wait({task}) + if task.cancelled(): + raise asyncio.CancelledError async def _phase_b(app: FastAPI) -> None: diff --git a/tests/test_startup_profile_reconcile_background.py b/tests/test_startup_profile_reconcile_background.py index 994278f9e..e4064cf05 100644 --- a/tests/test_startup_profile_reconcile_background.py +++ b/tests/test_startup_profile_reconcile_background.py @@ -204,3 +204,33 @@ async def scenario(): release_executor.set() asyncio.run(scenario()) + + +def test_shutdown_cancellation_stops_reconcile_dependent_warmups(monkeypatch): + entered = threading.Event() + release = threading.Event() + + def reconcile(): + entered.set() + release.wait(5) + return {} + + monkeypatch.setattr(importlib.import_module("services.performance_profiles"), + "reconcile_active_profile", reconcile) + + async def scenario(): + app = SimpleNamespace(state=SimpleNamespace()) + main._start_profile_reconcile(app) + try: + assert await asyncio.to_thread(entered.wait, 2) + warmup = asyncio.create_task(main._await_profile_reconcile(app)) + shutdown = asyncio.create_task(main._wait_for_profile_reconcile_thread(app)) + await asyncio.sleep(0.05) + assert warmup.cancelled() + assert not shutdown.done() + release.set() + await asyncio.wait_for(shutdown, 2) + finally: + release.set() + + asyncio.run(scenario()) From 54f8634fc221b97bfdc3695594a22e6fda3102bc Mon Sep 17 00:00:00 2001 From: Shimul Debnath <270455167+velixio@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:08:33 +0530 Subject: [PATCH 5/6] fix(asr): check cuDNN before selecting CTranslate2 CUDA at load time --- backend/services/asr_backend.py | 9 +++- docs/install/troubleshooting.md | 6 ++- tests/test_asr_cudnn_load_preflight.py | 62 ++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 tests/test_asr_cudnn_load_preflight.py diff --git a/backend/services/asr_backend.py b/backend/services/asr_backend.py index be476ea7d..9a2f66ce8 100644 --- a/backend/services/asr_backend.py +++ b/backend/services/asr_backend.py @@ -2964,7 +2964,14 @@ def _ctranslate2_cuda_ok() -> bool: # the #1529 crash. CPU always works. logger.warning("device probe failed — CTranslate2 taking the CPU path", exc_info=True) return False - return _cuda_reported_available() and not _rocm_torch() + # Availability checks are not a load-time guarantee: an already selected + # or directly constructed adapter can bypass them. Missing cuDNN aborts + # inside native CUDA code, so choose the supported CPU path before loading. + return ( + _cuda_reported_available() + and not _rocm_torch() + and _ctranslate2_cudnn_ok()[0] + ) def _auto_detect() -> str: diff --git a/docs/install/troubleshooting.md b/docs/install/troubleshooting.md index 439dcd933..121e9752e 100644 --- a/docs/install/troubleshooting.md +++ b/docs/install/troubleshooting.md @@ -565,8 +565,10 @@ restarted it, and the next attempt killed it again ([#1371](https://github.com/debpalash/VoiceStudio/issues/1371)). VoiceStudio now checks whether cuDNN 8 will load *before* selecting a CTranslate2 engine and falls back to PyTorch Whisper instead, so a missing library costs you WhisperX's -word-level alignment — not the backend. The repair below is still worth doing to -get WhisperX back. +word-level alignment — not the backend. Direct or already-selected CTranslate2 +adapters also check the CUDA libraries before loading and use their CPU path +when cuDNN is missing. Hosts with working CUDA libraries keep GPU acceleration. +The repair below is still worth doing to get WhisperX back. **Cause:** WhisperX and faster-whisper run on **CTranslate2**, which needs **cuDNN 8**, but PyTorch 2.8 ships cuDNN 9. VoiceStudio side-loads a cuDNN-8 copy diff --git a/tests/test_asr_cudnn_load_preflight.py b/tests/test_asr_cudnn_load_preflight.py new file mode 100644 index 000000000..deee7de9a --- /dev/null +++ b/tests/test_asr_cudnn_load_preflight.py @@ -0,0 +1,62 @@ +"""Direct ASR loads must avoid CUDA when its cuDNN runtime cannot load (#2904).""" +import importlib +import os +from pathlib import Path +import sys +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'backend')) + + +class CudnnLoadPreflightTests(unittest.TestCase): + def setUp(self): + self.ab = importlib.import_module('services.asr_backend') + self.device_caps = importlib.import_module('core.device_caps') + + def test_direct_faster_whisper_load_uses_cpu_only_when_cudnn_is_missing(self): + ab, device_caps = self.ab, self.device_caps + for cudnn_ok, expected in ((False, 'cpu'), (True, 'cuda')): + with self.subTest(cudnn_ok=cudnn_ok): + calls = [] + + def load(source, *, device, compute_type): + calls.append((source, device, compute_type)) + return object() + + with patch.dict(sys.modules, {'faster_whisper': SimpleNamespace(WhisperModel=load)}), \ + patch.dict(os.environ, {'ASR_COMPUTE_TYPE': ''}), \ + patch.object(device_caps, 'detect_host_caps', return_value=SimpleNamespace(family='cuda')), \ + patch.object(ab, '_cuda_reported_available', return_value=True), \ + patch.object(ab, '_rocm_torch', return_value=False), \ + patch.object(ab, '_ctranslate2_cudnn_ok', return_value=(cudnn_ok, 'test')), \ + patch.object(ab, '_ctranslate2_execstack_ok', return_value=(True, 'ready')), \ + patch.object(ab, '_local_model_source', return_value='/local/installed-whisper'): + backend = ab.FasterWhisperBackend(model_name='installed-whisper') + backend._ensure_model() + self.assertEqual(calls, [('/local/installed-whisper', expected, + 'int8' if expected == 'cpu' else 'float16')]) + self.assertIsNotNone(backend._model) + self.assertEqual(ab.WhisperXBackend._pick_device(), + (expected, 'int8' if expected == 'cpu' else 'float16')) + + def test_cpu_override_never_probes_cuda_libraries(self): + ab, device_caps = self.ab, self.device_caps + with patch.object(device_caps, 'detect_host_caps', return_value=SimpleNamespace(family='cpu')), \ + patch.object(ab, '_ctranslate2_cudnn_ok') as probe: + self.assertFalse(ab._ctranslate2_cuda_ok()) + probe.assert_not_called() + + def test_rocm_never_probes_nvidia_libraries(self): + ab, device_caps = self.ab, self.device_caps + with patch.object(device_caps, 'detect_host_caps', return_value=SimpleNamespace(family='cuda')), \ + patch.object(ab, '_cuda_reported_available', return_value=True), \ + patch.object(ab, '_rocm_torch', return_value=True), \ + patch.object(ab, '_ctranslate2_cudnn_ok') as probe: + self.assertFalse(ab._ctranslate2_cuda_ok()) + probe.assert_not_called() + + +if __name__ == '__main__': + unittest.main() From 2b64a935405e67ebf0ca43763788f8b1cf4510ab Mon Sep 17 00:00:00 2001 From: Shimul Debnath <270455167+velixio@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:22:04 +0530 Subject: [PATCH 6/6] fix(startup): skip completed reconciliation handles from closed loops --- backend/main.py | 5 +++-- ...est_startup_profile_reconcile_background.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/backend/main.py b/backend/main.py index 2b26f9770..d17f0a667 100644 --- a/backend/main.py +++ b/backend/main.py @@ -948,7 +948,7 @@ async def _wait_for_profile_reconcile_thread(app: FastAPI) -> None: task.cancel() await asyncio.gather(task, return_exceptions=True) worker = getattr(app.state, "profile_reconcile_worker", None) - if worker is not None: + if worker is not None and not worker.done(): # asyncio.wait neither cancels the executor future on timeout nor # needs a free executor thread to enforce the shutdown deadline. await asyncio.wait({worker}, timeout=20.0) @@ -958,7 +958,8 @@ async def _await_profile_reconcile(app: FastAPI) -> None: """Let model warm-ups wait for reconciliation without inheriting its failure.""" task = getattr(app.state, "profile_reconcile_task", None) if task is not None: - await asyncio.wait({task}) + if not task.done(): + await asyncio.wait({task}) if task.cancelled(): raise asyncio.CancelledError diff --git a/tests/test_startup_profile_reconcile_background.py b/tests/test_startup_profile_reconcile_background.py index e4064cf05..fe9b5d537 100644 --- a/tests/test_startup_profile_reconcile_background.py +++ b/tests/test_startup_profile_reconcile_background.py @@ -234,3 +234,21 @@ async def scenario(): release.set() asyncio.run(scenario()) + + +def test_completed_reconcile_handles_from_closed_loop_do_not_break_shutdown(): + loop = asyncio.new_event_loop() + + async def complete(): + return None + + task = loop.create_task(complete()) + loop.run_until_complete(task) + worker = loop.create_future() + worker.set_result(None) + loop.close() + app = SimpleNamespace(state=SimpleNamespace( + profile_reconcile_task=task, profile_reconcile_worker=worker, + )) + asyncio.run(main._wait_for_profile_reconcile_thread(app)) + asyncio.run(main._await_profile_reconcile(app))