diff --git a/Dockerfile b/Dockerfile index 0b57ff4..6914507 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,6 @@ WORKDIR /app ENV DEBIAN_FRONTEND=noninteractive ENV PYTHONUNBUFFERED=1 ENV ENABLE_FLASHSR=true - ENV TORCH_CUDA_ARCH_LIST="8.0;8.6;8.9;9.0" ENV MAX_JOBS=4 @@ -14,23 +13,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && add-apt-repository ppa:deadsnakes/ppa \ && apt-get update \ && apt-get install -y --no-install-recommends \ - python3.11 \ - python3.11-dev \ - python3.11-venv \ - python3.11-distutils \ - git \ - curl \ - ca-certificates \ - build-essential \ - ninja-build \ - libsndfile1 \ - ffmpeg \ + python3.11 python3.11-dev python3.11-venv python3.11-distutils \ + git curl ca-certificates build-essential ninja-build libsndfile1 ffmpeg \ && rm -rf /var/lib/apt/lists/* \ && ln -sf /usr/bin/python3.11 /usr/bin/python3 \ && ln -sf /usr/bin/python3 /usr/bin/python RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 - RUN curl -LsSf https://astral.sh/uv/install.sh | sh ENV PATH="/root/.local/bin:$PATH" @@ -38,28 +27,20 @@ COPY pyproject.toml ./ COPY .python-version ./ COPY README.md ./ COPY main.py ./ - +COPY runner ./runner COPY third_party/VibeVoice ./third_party/VibeVoice COPY overrides ./overrides COPY scripts ./scripts RUN mkdir -p models - RUN uv venv --python /usr/bin/python3.11 - -RUN . .venv/bin/activate && uv sync - +RUN . .venv/bin/activate && uv sync --extra vibevoice +RUN . .venv/bin/activate && uv pip install -e third_party/VibeVoice RUN . .venv/bin/activate && uv pip install --no-build-isolation flash-attn -RUN if [ -f "overrides/app.py" ]; then \ - cp overrides/app.py third_party/VibeVoice/demo/web/app.py; \ - fi -RUN if [ -f "overrides/index.html" ]; then \ - cp overrides/index.html third_party/VibeVoice/demo/web/index.html; \ - fi +RUN if [ -f overrides/app.py ]; then cp overrides/app.py third_party/VibeVoice/demo/web/app.py; fi +RUN if [ -f overrides/index.html ]; then cp overrides/index.html third_party/VibeVoice/demo/web/index.html; fi EXPOSE 8000 - ENV HF_HOME=/app/models - CMD [".venv/bin/python", "scripts/run_realtime_demo.py", "--port", "8000", "--host", "0.0.0.0"] diff --git a/README.md b/README.md index d8ee653..18fe1a6 100644 --- a/README.md +++ b/README.md @@ -1,410 +1,294 @@ -# šŸŽ™ļø VibeVoice Realtime Runner +# VibeVoice Realtime FASTAPI -
+A local, OpenAI-compatible text-to-speech service with two platform-specific runtime paths: -![VibeVoice](https://img.shields.io/badge/VibeVoice-Realtime-blue?style=for-the-badge) -![Python](https://img.shields.io/badge/Python-3.11-yellow?style=for-the-badge&logo=python) -![FastAPI](https://img.shields.io/badge/FastAPI-0.109-green?style=for-the-badge&logo=fastapi) -![OpenAI API](https://img.shields.io/badge/OpenAI_API-Compatible-orange?style=for-the-badge&logo=openai) +- **Linux/NVIDIA:** Microsoft VibeVoice through the existing PyTorch/CUDA realtime demo. +- **Apple Silicon MacBooks:** Qwen3-TTS through native MLX backends, including the exact linked 1.7B 8-bit model and a matching 1.7B 4-bit model. -**A high-performance local runner for Microsoft's VibeVoice text-to-speech models.** -*Now with OpenAI-compatible API endpoints and multi-model support!* +The server remains local and does not require a hosted inference provider. -[Features](#features) ā€¢ [Quick Start](#quick-start) ā€¢ [Multi-Model Support](#-multi-model-support) ā€¢ [API Documentation](#api-documentation) ā€¢ [Credits](#credits) +## Supported model profiles -
+| Model key | Model source | Runtime | Platform | Notes | +|---|---|---|---|---| +| `realtime-0.5b` | `microsoft/VibeVoice-Realtime-0.5B` | Existing VibeVoice subprocess | Linux/CUDA, PyTorch fallback devices | Realtime websocket and web UI | +| `tts-1.5b` | `microsoft/VibeVoice-1.5B` | Native long-form adapter | Backend-dependent | Non-streaming | +| `tts-7b` | `vibevoice/VibeVoice-7B` by default | Native long-form adapter | Backend-dependent | Non-streaming | +| `qwen3-tts-mlx-8bit` | `aufklarer/Qwen3-TTS-12Hz-1.7B-Base-MLX-8bit` | Speech Swift CLI | Apple Silicon macOS | Exact requested 8-bit/minmax bundle, 24 kHz | +| `qwen3-tts-mlx-4bit` | `mlx-community/Qwen3-TTS-12Hz-1.7B-Base-4bit` | `mlx-audio` in process | Apple Silicon macOS | 4-bit/affine bundle, 24 kHz | ---- +The two Qwen backends are deliberately separate. The aufklarer 8-bit bundle is packaged for Speech Swift's minmax loader, while the registered 4-bit bundle is packaged for `mlx-audio`'s affine loader. Treating them as interchangeable produces configuration or tensor-loading failures. -## šŸš€ Features +List every canonical key and alias: -- **Local & Private**: Runs entirely on your machine (CUDA/MPS/CPU). -- **Realtime Streaming**: Low-latency text-to-speech generation. -- **Multi-Model Support**: Registry for Realtime 0.5B, 1.5B, and 7B models. -- **LavaSR Super-Resolution**: Neural audio upsampling (24kHz ā†’ 48kHz) at 300-500x realtime, enabled by default. Surpasses 6GB diffusion models in quality. -- **OpenAI API Compatible**: Drop-in replacement for OpenAI's TTS API with model selection. -- **Multiple Audio Formats**: Supports Opus (default), WAV, and MP3 output. -- **Web Interface**: Built-in interactive demo UI. -- **Multi-Platform**: Optimized for Ubuntu (CUDA) and macOS (Apple Silicon). -- **Easy Setup**: Powered by `uv` for fast, reliable dependency management. +```bash +uv run vibevoice-server --list-models +``` -## āš” Quick Start +## Apple Silicon MacBook setup -### Prerequisites +### Requirements -- **uv** installed: `curl -LsSf https://astral.sh/uv/install.sh | sh` -- **Git** -- **Hugging Face Account** (for model download) +- Apple Silicon (`arm64`) Mac; do not run the server under Rosetta. +- macOS 15 or newer for the current Speech Swift package. +- Native ARM Homebrew at `/opt/homebrew`. +- `uv`. +- Enough unified memory for the selected model and runtime. The 4-bit profile is the lower-memory option. -### Installation +### Install -1. **Bootstrap the environment**: - ```bash - ./scripts/bootstrap_uv.sh - ``` +```bash +git clone https://github.com/groxaxo/vibevoice-realtimeFASTAPI.git +cd vibevoice-realtimeFASTAPI +bash scripts/bootstrap_macos.sh +``` -2. **Download a model**: - ```bash - # Default: Realtime 0.5B (fully supported) - uv run python scripts/download_model.py +The bootstrap script: - # Or specify a model explicitly - uv run python scripts/download_model.py --model realtime-0.5b - uv run python scripts/download_model.py --model tts-1.5b - uv run python scripts/download_model.py --model tts-7b - ``` +1. Uses a Python 3.12 `uv` environment. +2. Installs the mutually isolated Python `mac` extra, including the model-compatible `mlx-audio==0.3.0` loader. +3. Installs Speech Swift for the exact 8-bit model. +4. Installs `ffmpeg` for optional MP3 and Opus responses. -3. **Run the server**: - ```bash - # Using the generic launcher - uv run python scripts/run_server.py --model realtime-0.5b --port 8000 +The Qwen-only path does **not** import PyTorch or require the VibeVoice submodule. The `mac` and `vibevoice` extras are declared mutually exclusive because their validated Transformers versions are incompatible. + +### Run the exact 8-bit model + +```bash +uv run vibevoice-server \ + --model qwen3-tts-mlx-8bit \ + --host 127.0.0.1 \ + --port 8000 +``` - # Or the backward-compatible script - uv run python scripts/run_realtime_demo.py --port 8000 - ``` +This adapter invokes Speech Swift with the exact model ID: - - **Web UI**: Open [http://127.0.0.1:8000/web](http://127.0.0.1:8000/web) - - **API**: `http://127.0.0.1:8000/v1/audio/speech` +```text +aufklarer/Qwen3-TTS-12Hz-1.7B-Base-MLX-8bit +``` -## šŸ§© Multi-Model Support +### Run the 4-bit model -This runner supports multiple VibeVoice model families through a unified registry and adapter layer. +```bash +uv run vibevoice-server \ + --model qwen3-tts-mlx-4bit \ + --host 127.0.0.1 \ + --port 8000 +``` -### Supported Models +On first use, `mlx-audio` downloads: -| Model Key | HF Model ID | Family | Status | Streaming | Multi-Speaker | -|:---|:---|:---|:---|:---|:---| -| `realtime-0.5b` | `microsoft/VibeVoice-Realtime-0.5B` | Realtime | āœ… Fully supported | āœ… | āŒ | -| `tts-1.5b` | `microsoft/VibeVoice-1.5B` | Longform | šŸ”§ Registry + API ready | āŒ | āœ… | -| `tts-7b` | `microsoft/VibeVoice-7B` | Longform | šŸ”§ Registry + API ready | āŒ | āœ… | +```text +mlx-community/Qwen3-TTS-12Hz-1.7B-Base-4bit +``` -### Model Aliases +The 4-bit loader resolves its source in this order: -For backward compatibility and convenience, the following aliases are supported: +1. Explicit `--model-path` value, which may be a local directory or Hugging Face repository ID. +2. A non-empty registry-local directory under `models/`. +3. The registry Hugging Face ID. -| Alias | Resolves To | -|:---|:---| -| `tts-1` | `realtime-0.5b` | -| `tts-1-hd` | `realtime-0.5b` | -| `vibevoice-realtime-0.5b` | `realtime-0.5b` | -| `vibevoice-1.5b` | `tts-1.5b` | -| `vibevoice-7b` | `tts-7b` | +That fallback fixes the previous launcher behavior that converted every source into a local `Path`, even when no local model directory existed. -### Model Selection in API Requests +### Basic synthesis request -The `model` field in `/v1/audio/speech` requests is **no longer ignored**. It is resolved via the alias mapping and validated for capability: +Use WAV while validating the installation because it requires no transcoding: ```bash -# Uses realtime-0.5b (backward compatible) curl http://127.0.0.1:8000/v1/audio/speech \ - -H "Content-Type: application/json" \ - -d '{"model": "tts-1", "input": "Hello!", "voice": "en-Carter_man"}' + -H 'Content-Type: application/json' \ + -d '{ + "model": "qwen3-tts-mlx-8bit", + "input": "Hello from a Qwen3 TTS model running locally on a MacBook.", + "voice": "default", + "language": "english", + "response_format": "wav" + }' \ + --output speech.wav +``` -# Explicitly request realtime -curl http://127.0.0.1:8000/v1/audio/speech \ - -H "Content-Type: application/json" \ - -d '{"model": "realtime-0.5b", "input": "Hello!", "voice": "en-Carter_man"}' +For the 4-bit server, change only the model value: -# Request a longform model (returns 501 if backend not installed) -curl http://127.0.0.1:8000/v1/audio/speech \ - -H "Content-Type: application/json" \ - -d '{"model": "tts-1.5b", "input": "Hello!"}' +```json +"model": "qwen3-tts-mlx-4bit" ``` -### Error Responses +### Reference-voice conditioning -| Status | Condition | -|:---|:---| -| **400** | Invalid request for model (e.g., `speakers` sent to a realtime model) | -| **404** | Unknown model key/alias | -| **501** | Known model but backend not installed | +The Base profiles do not expose named speaker presets. Supply a local reference-audio path that is readable by the server process: -### Longform Model Status +```bash +curl http://127.0.0.1:8000/v1/audio/speech \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "qwen3-tts-mlx-4bit", + "input": "This line uses the reference speaker characteristics.", + "language": "english", + "ref_audio": "/absolute/path/to/reference.wav", + "response_format": "wav" + }' \ + --output cloned.wav +``` -The `tts-1.5b` and `tts-7b` models are registered in the model registry with full API plumbing, but require a compatible long-form inference backend to be installed. When requested without a backend, the API returns a clear 501 error: +For the `mlx-audio` 4-bit profile, adding the matching transcript enables its ICL voice-cloning path: ```json { - "error": { - "message": "Model 'tts-1.5b' is registered but no compatible long-form backend is installed/configured.", - "type": "backend_unavailable" - } + "ref_audio": "/absolute/path/to/reference.wav", + "ref_text": "Transcript of the reference recording" } ``` -### WebSocket Streaming (`/stream`) - -The `/stream` WebSocket endpoint is **realtime-only**. Longform models are not supported on this endpoint and will be rejected with an error message. - -### šŸŒ Frontpage Controls (Web UI) - -The frontpage at `/web` (also available at `/`) is fully connected to backend endpoints and exposes: - -- **Model selection** (`tts-1`, `tts-1-hd`) for OpenAI-compatible requests -- **Voice selection** from `GET /config` and `GET /v1/audio/voices` -- **Temperature control** (`temp`): set `0` to disable sampling -- **Generate Audio** action that calls `POST /v1/audio/speech` -- **Download Audio** action to export the generated file in the selected format (`opus`/`wav`/`mp3`) - -## šŸ“– API Documentation +`ref_audio` is a server-local path, not an upload URL. Do not expose this service to untrusted clients without authentication and path controls. -This runner provides OpenAI-compatible endpoints for easy integration with existing tools and libraries. +## Linux/NVIDIA VibeVoice setup -### šŸ—£ļø Speech Generation - -**Endpoint**: `POST /v1/audio/speech` - -Generates audio from text with LavaSR super-resolution enabled by default (24kHz ā†’ 48kHz). -This is also the endpoint used by the frontpage "Generate Audio" button. +Clone the submodule, resolve the VibeVoice extra, and install the populated checkout editable: ```bash -curl http://127.0.0.1:8000/v1/audio/speech \ - -H "Content-Type: application/json" \ - -d '{ - "model": "tts-1", - "input": "Hello, this is VibeVoice running locally!", - "voice": "en-Carter_man", - "response_format": "opus" - }' \ - --output speech.opus +git clone --recurse-submodules https://github.com/groxaxo/vibevoice-realtimeFASTAPI.git +cd vibevoice-realtimeFASTAPI +./scripts/bootstrap_uv.sh +uv run python scripts/download_model.py --model realtime-0.5b ``` -| Parameter | Type | Description | -| :--- | :--- | :--- | -| `model` | `string` | Model identifier. Resolved via alias mapping (see [Multi-Model Support](#-multi-model-support)). Default: `tts-1` ā†’ `realtime-0.5b`. | -| `input` | `string` | The text to generate audio for. | -| `voice` | `string` | The voice ID to use (see `/v1/audio/voices`). | -| `response_format` | `string` | Output format: `opus` (default, 48kHz), `wav`, or `mp3`. | -| `temp` | `float` | Sampling temperature. When provided (>0), enables sampling with the given temperature. | -| `speed` | `float` | Speed of generation (currently ignored). | -| `speakers` | `list` | Multi-speaker dialogue turns (longform models only). | - -### šŸŽ¤ List Voices +Start the existing realtime application: -**Endpoint**: `GET /v1/audio/voices` +```bash +CUDA_VISIBLE_DEVICES=0 uv run vibevoice-server \ + --model realtime-0.5b \ + --host 0.0.0.0 \ + --port 8000 +``` -Returns a list of available voices. +The legacy launcher remains available: ```bash -curl http://127.0.0.1:8000/v1/audio/voices +uv run python scripts/run_realtime_demo.py --port 8000 ``` -**Response:** -```json -{ - "voices": [ - { - "id": "en-Carter_man", - "name": "en-Carter_man", - "object": "voice", - "category": "vibe_voice", - ... - }, - ... - ] -} +For CUDA builds that need FlashAttention outside Docker: + +```bash +uv sync --extra vibevoice +uv pip install --no-build-isolation flash-attn ``` -### ā¤ļø Health Check +## Server architecture -**Endpoint**: `GET /health` +There are now two serving paths rather than one application pretending every adapter is the realtime engine: -Returns service readiness information including lazy loading status, active model, and adapter type. +1. **Realtime VibeVoice:** `RealtimeDemoAdapter` launches the existing vendored demo and preserves `/stream`, `/web`, and the optimized CUDA path. +2. **Native adapter API:** Qwen3-TTS and non-realtime adapters run through `runner.api`, which calls the selected adapter directly. -### āš™ļø Server Configuration +A process binds to one active model. Requests naming another registered model receive HTTP `409` with a command showing how to start the correct instance. This prevents accidental requests from being synthesized by the wrong loaded model. -**Endpoint**: `GET /config` +## Native API endpoints -Returns available voices, models, aliases, and per-model capabilities. +| Endpoint | Purpose | +|---|---| +| `GET /` | Service and endpoint summary | +| `GET /health` | Backend availability, active source, and load state | +| `GET /config` | Complete registry plus active model capabilities | +| `GET /v1/models` | Active OpenAI-style model row | +| `GET /v1/audio/voices` | Voices/conditioning modes for the active adapter | +| `POST /v1/audio/speech` | OpenAI-style speech synthesis | +| `GET /docs` | FastAPI/OpenAPI documentation | -## āš™ļø Configuration +### Qwen request fields -### Device Selection +| Field | Type | Description | +|---|---|---| +| `model` | string | Active model key or alias | +| `input` | string | Text to synthesize | +| `voice` | string | `default`, or a local reference WAV path for Base profiles | +| `language` | string | `english`, `chinese`, `auto`, and other model-supported language names | +| `response_format` | string | `wav`, `mp3`, or `opus`; WAV is the native output | +| `temp` | float | Sampling temperature | +| `top_k` | integer | Top-k sampling cutoff | +| `top_p` | float | Nucleus sampling threshold; used by the 4-bit `mlx-audio` backend | +| `max_tokens` | integer | Maximum generated codec tokens | +| `repetition_penalty` | float | Used by the 4-bit `mlx-audio` backend | +| `ref_audio` | string | Absolute/local reference-audio path | +| `ref_text` | string | Optional reference transcript for 4-bit ICL cloning | -The runner automatically detects the best available device: -- **CUDA**: NVIDIA GPUs (Linux) -- **MPS**: Apple Silicon (macOS) -- **CPU**: Fallback +Streaming is not yet exposed by the native Qwen HTTP adapter. A request with `stream: true` is rejected instead of silently returning a buffered response. -To force a specific device: -```bash -uv run python scripts/run_realtime_demo.py --device cpu -``` +## Configuration -### Inference Steps +### Model source override -Specify the number of DDPM inference steps. Lower values reduce latency and improve realtime responsiveness. The default is **5** (official realtime profile). +The existing argument name is retained for compatibility: ```bash -uv run python scripts/run_realtime_demo.py --inference-steps 5 +uv run vibevoice-server --model qwen3-tts-mlx-4bit \ + --model-path /absolute/path/to/local/model ``` -### Custom Model Path +It can also be an alternate Hugging Face ID for the `mlx-audio` profile: ```bash -uv run python scripts/run_realtime_demo.py --model-path /path/to/model +uv run vibevoice-server --model qwen3-tts-mlx-4bit \ + --model-path organization/compatible-qwen3-tts-mlx-model ``` -### LavaSR Audio Super-Resolution +For the Speech Swift 8-bit profile, use `--model-path` only as a Hugging Face model-ID override. -LavaSR is **enabled by default** to upsample audio from 24kHz to 48kHz using neural network bandwidth extension. This provides studio-quality 48kHz audio output with minimal performance impact (~2ms per chunk). +### Concurrency -To disable LavaSR (output will be 24kHz): -```bash -export ENABLE_LAVASR=false -uv run python scripts/run_realtime_demo.py -``` +MLX generation is serialized internally because model instances are not thread-safe. The API also defaults to one concurrent synthesis request: -Or enable it explicitly: ```bash -export ENABLE_LAVASR=true -uv run python scripts/run_realtime_demo.py +uv run vibevoice-server --model qwen3-tts-mlx-4bit \ + --max-concurrent-requests 1 ``` -**Benefits of LavaSR:** -- Neural network super-resolution (not simple interpolation) -- 300-500x realtime speed (~2ms latency per chunk) -- Higher quality 48kHz audio output -- Direct 24kHz ā†’ 48kHz upsampling (no quality loss) -- Quality surpasses 6GB diffusion models (best LSD scores) -- Compatible with Opus format for optimal compression - -**Benchmark Results (RTX 3090):** -| Chunk Duration | Upsampling Time | Speed | -|----------------|-----------------|-------| -| 0.25s | 1.9ms | 128x realtime | -| 0.50s | 1.9ms | 263x realtime | -| 1.00s | 1.9ms | 523x realtime | -| 2.00s | 2.1ms | 961x realtime | - -## šŸ“Š Realtime Benchmarking (`/stream`) - -Use the websocket benchmark script to measure TTFA, chunk pacing, and RTF with reproducible settings. - -```bash -uv run python scripts/benchmark_stream_endpoint.py \ - --ws-url ws://127.0.0.1:8000/stream \ - --voice en-Carter_man \ - --runs 10 \ - --temp 0 \ - --steps 5 -``` +Increasing this value does not create independent model replicas. -The script writes a JSON report to `/tmp` by default and can compare against a prior run using `--baseline-json`. +### Environment variables -## šŸš€ Production Deployment +| Variable | Purpose | +|---|---| +| `TTS_MODEL` | Active model used by the Uvicorn factory/reload mode | +| `MODEL_PATH` | Model source override | +| `MODEL_DEVICE` | Device override for PyTorch adapters | +| `MAX_CONCURRENT_REQUESTS` | Native API admission limit | +| `SPEECH_SWIFT_BIN` | Explicit path to `speech` or legacy `audio` executable | +| `QWEN3_TTS_MODEL_ID` | Exact 8-bit Speech Swift model-ID override | +| `QWEN3_TTS_TIMEOUT_SECONDS` | Speech Swift subprocess timeout; default 1800 | +| `HF_HOME` | Hugging Face cache root | -**Important:** This is a TTS-only service. Whisper transcription is **not** automatically launched. Whisper endpoints (if needed for validation) must be run separately. +## Local validation -### Starting the Server +No GitHub Actions workflow is required. Run checks on the target machine: ```bash -# Recommended: Use the provided script with GPU selection -CUDA_VISIBLE_DEVICES=2 uv run python scripts/run_realtime_demo.py --port 8000 - -# Or run the demo directly -CUDA_VISIBLE_DEVICES=2 uv run python third_party/VibeVoice/demo/vibevoice_realtime_demo.py \ - --port 8000 \ - --model_path models/VibeVoice-Realtime-0.5B \ - --device cuda \ - --inference_steps 5 +uv sync --extra dev --extra mac # Apple Silicon +# uv sync --extra dev --extra vibevoice # Linux/VibeVoice +uv run ruff check runner scripts test_runner.py test_macos_mlx.py +uv run pytest test_macos_mlx.py # hardware-free Mac integration tests +# uv run pytest # full suite in a VibeVoice environment +python -m compileall -q runner scripts main.py +bash -n scripts/bootstrap_uv.sh scripts/bootstrap_macos.sh ``` -**Note:** Replace `CUDA_VISIBLE_DEVICES=2` with your available GPU. Check GPU availability with `nvidia-smi`. +A real synthesis smoke test must run on Apple Silicon because Linux cannot execute MLX/Metal or Speech Swift. -### Boot Autostart with Lazy Load +## Docker -To install a systemd service that binds on all interfaces, listens on port `8881`, and defers model initialization until the first speech request: +The Dockerfile remains CUDA-specific and installs the `vibevoice` dependency extra. MLX is an Apple framework and is not supported inside the NVIDIA Linux image. ```bash -CUDA_VISIBLE_DEVICES=3 HOST=0.0.0.0 PORT=8881 ./scripts/install_systemd_service.sh -sudo systemctl start vibevoice-realtime.service +docker build -t vibevoice-realtime . +docker run --gpus all -p 8000:8000 vibevoice-realtime ``` -This exposes the UI at `http://:8881/web`, keeps the OpenAI-compatible API under `/v1/...`, and sets `ENABLE_LAZY_LOAD=true` with `ENABLE_STARTUP_WARMUP=false` for fast boot-time startup. Adjust `CUDA_VISIBLE_DEVICES` if you want to pin the service to a different GPU. - -### Restarting with New Code - -After pulling updates, restart the server to apply changes: +## License and upstream projects -```bash -# Find and kill existing process -ps aux | grep vibevoice_realtime_demo -kill - -# Restart with new code -CUDA_VISIBLE_DEVICES=2 uv run python scripts/run_realtime_demo.py --port 8000 -``` +Review the licenses and model cards for the software and model weights you use: -## šŸ”§ Recommended Concurrency - -Based on end-to-end benchmarks (TTS + Whisper transcription), the recommended default concurrency is **2 concurrent requests**. - -**Benchmark Results (RTX 3090, 5 inference steps):** - -| Concurrency | TTS avg/p95 (s) | Whisper avg/p95 (s) | E2E avg (s) | Throughput (req/s) | -|-------------|-----------------|---------------------|-------------|-------------------| -| 2 | 5.57 / 9.11 | 0.39 / 0.66 | 5.96 | 0.333 | -| 4 | 11.15 / 14.55 | 0.43 / 0.82 | 11.58 | 0.324 | -| 8 | 20.86 / 27.11 | 0.43 / 0.81 | 21.29 | 0.322 | - -**Key Findings:** -- TTS is the bottleneck; Whisper adds minimal latency (~0.3-0.4s) regardless of concurrency -- Throughput plateaus at ~0.32-0.33 req/s beyond 2 concurrent requests -- Latency increases significantly with higher concurrency due to TTS queueing -- Single-stream RTF: ~0.39 (2.6x faster than realtime) -- Recommended max concurrent requests: **2** for optimal latency/throughput balance - -## šŸŽ§ Demos - -All examples generated using **15 inference steps** with text in the voice's native language. - -### English -| Voice | Audio Example (MP3) | -| :--- | :--- | -| **en-Carter_man** | | -| **en-Davis_man** | | -| **en-Emma_woman** | | -| **en-Frank_man** | | -| **en-Grace_woman** | | -| **en-Mike_man** | | -| **in-Samuel_man** | | - -### Other Languages -| Language | Voice | Audio Example (MP3) | -| :--- | :--- | :--- | -| **German** | de-Spk0_man | | -| **German** | de-Spk1_woman | | -| **Spanish** | sp-Spk0_woman | | -| **Spanish** | sp-Spk1_man | | -| **French** | fr-Spk0_man | | -| **French** | fr-Spk1_woman | | -| **Italian** | it-Spk0_woman | | -| **Italian** | it-Spk1_man | | -| **Japanese** | jp-Spk0_man | | -| **Japanese** | jp-Spk1_woman | | -| **Korean** | kr-Spk0_woman | | -| **Korean** | kr-Spk1_man | | -| **Dutch** | nl-Spk0_man | | -| **Dutch** | nl-Spk1_woman | | -| **Polish** | pl-Spk0_man | | -| **Polish** | pl-Spk1_woman | | -| **Portuguese** | pt-Spk0_woman | | -| **Portuguese** | pt-Spk1_man | | - - -## šŸ† Credits & Acknowledgements - -This project stands on the shoulders of giants. Huge thanks to: - -- **[Microsoft](https://github.com/microsoft/VibeVoice)**: For releasing the incredible **VibeVoice** model and the original codebase. -- **[ysharma3501/LavaSR](https://github.com/ysharma3501/LavaSR)**: For the high-quality neural audio super-resolution model. -- **[groxaxo](https://github.com/groxaxo)**: For the original repository and initial setup. -- **[Kokoro FastAPI Creators](https://github.com/remsky/Kokoro-FastAPI)**: For inspiration on the FastAPI implementation and structure. -- **Open Source Community**: For all the tools and libraries that make this possible. - ---- - -
-Made with ā¤ļø for the AI Community -
+- Microsoft VibeVoice +- Qwen3-TTS +- Speech Swift +- MLX / `mlx-audio` +- The selected Hugging Face model repository diff --git a/main.py b/main.py index bde1629..2457eaa 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,6 @@ -def main(): - print("Hello from vibevoice!") +"""Installed entry point for the multi-model TTS server.""" + +from runner.cli import main if __name__ == "__main__": diff --git a/pyproject.toml b/pyproject.toml index eca196d..d83f1d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,43 +4,65 @@ build-backend = "setuptools.build_meta" [project] name = "vibevoice-runner" -version = "0.1.0" -description = "Local TTS runner for Microsoft VibeVoice - Open-WebUI compatible, privacy-first with uv" +version = "0.2.0" +description = "Local OpenAI-compatible TTS runner for VibeVoice and Qwen3-TTS on CUDA and Apple Silicon" readme = "README.md" requires-python = ">=3.11,<3.13" dependencies = [ - # Keep HF Hub <1.0 to satisfy transformers' constraints used by VibeVoice. - "huggingface_hub>=0.30.0,<1.0", - # VibeVoice streaming path is validated against 4.51.3; newer releases regress KV cache masking. + "fastapi>=0.115.0,<1.0", + "uvicorn[standard]>=0.30.0,<1.0", + "numpy>=1.26.0,<3.0", + "scipy>=1.12.0,<2.0", + "pydub>=0.25.1,<1.0", +] + +[project.optional-dependencies] +# PyTorch/VibeVoice runtime. scripts/bootstrap_uv.sh installs the populated +# third_party/VibeVoice checkout editable after uv resolves this dependency set. +vibevoice = [ + "huggingface-hub>=0.30.0,<1.0", "transformers==4.51.3", - # Ensure uv manages VibeVoice + its dependency set (prevents uv from "undoing" the env). "inflect>=7.5.0", - "vibevoice", - # Audio processing "librosa>=0.10.0", - # LavaSR super-resolution dependencies "vocos>=0.1.0", "einops>=0.8.0", ] - -[project.optional-dependencies] +# Pin the loader version used to create the registered 4-bit model bundle. +# The exact aufklarer 8-bit bundle uses Speech Swift, installed separately by +# scripts/bootstrap_macos.sh. Transformers 5 requires the Hub 1.x client. +mac = [ + "huggingface-hub>=1.3.0,<2.0; sys_platform == 'darwin' and platform_machine == 'arm64'", + "mlx-audio==0.3.0; sys_platform == 'darwin' and platform_machine == 'arm64'", +] cuda = [ - "flash-attn>=2.0.0", + "flash-attn>=2.0.0; sys_platform == 'linux' and platform_machine == 'x86_64'", ] dev = [ + "httpx>=0.27.0", "pytest>=7.0.0", "ruff>=0.1.0", "mypy>=1.0.0", ] +[project.scripts] +vibevoice-server = "runner.cli:main" + [tool.setuptools] -# Prevent setuptools from trying to treat top-level folders like `models/` and `third_party/` -# as importable packages when uv installs this project. +# Prevent setuptools from treating top-level models/ and third_party/ as packages. py-modules = ["main"] packages = ["runner", "runner.adapters"] -[tool.uv.sources] -vibevoice = { path = "third_party/VibeVoice", editable = true } +[tool.uv] +# mlx-audio 0.3 uses Transformers 5 RC + huggingface-hub 1.x while the +# validated VibeVoice path pins Transformers 4.51.3 + huggingface-hub <1.0. +# They are separate platform runtimes and cannot coexist in one environment, +# so resolve their extras in independent forks. +conflicts = [ + [ + { extra = "vibevoice" }, + { extra = "mac" }, + ], +] [tool.ruff] line-length = 100 @@ -55,3 +77,8 @@ python_version = "3.11" warn_return_any = true warn_unused_configs = true ignore_missing_imports = true + +[tool.pytest.ini_options] +testpaths = ["."] +python_files = ["test_*.py"] +addopts = "-q" diff --git a/runner/adapter_factory.py b/runner/adapter_factory.py index 7a5ba7a..36a723c 100644 --- a/runner/adapter_factory.py +++ b/runner/adapter_factory.py @@ -1,29 +1,39 @@ -"""Factory ā€“ instantiates the correct adapter for a given model profile.""" +"""Factory ā€“ instantiate the correct adapter for a model profile. + +Backend imports are intentionally lazy. A Qwen3/MLX-only installation on a Mac +must not import PyTorch or the vendored VibeVoice package merely to start the +server, and a CUDA installation must not require Apple's MLX runtime. +""" from __future__ import annotations from typing import Any from runner.adapters.base import EngineAdapter -from runner.adapters.longform_native import LongformNativeAdapter -from runner.adapters.realtime_demo import RealtimeDemoAdapter from runner.errors import UnknownModelError from runner.model_registry import ModelProfile, get_model_profile def make_adapter(model_key: str, **kwargs: Any) -> EngineAdapter: - """Create an :class:`EngineAdapter` for the given canonical *model_key*. + """Create an :class:`EngineAdapter` for canonical *model_key*.""" + profile: ModelProfile = get_model_profile(model_key, include_native=True) - Extra *kwargs* are forwarded to the adapter constructor (e.g. - ``model_path``, ``device``). - """ - profile: ModelProfile = get_model_profile(model_key) + if profile.loader_mode == "subprocess_demo": + from runner.adapters.realtime_demo import RealtimeDemoAdapter - if profile.family == "realtime" and profile.loader_mode == "subprocess_demo": return RealtimeDemoAdapter(profile, **kwargs) + if profile.loader_mode == "native_longform": + from runner.adapters.longform_native import LongformNativeAdapter - if profile.family == "longform" and profile.loader_mode == "native_longform": return LongformNativeAdapter(profile, **kwargs) + if profile.loader_mode == "speech_swift_cli": + from runner.adapters.qwen3_mlx import Qwen3SpeechSwiftAdapter + + return Qwen3SpeechSwiftAdapter(profile, **kwargs) + if profile.loader_mode == "mlx_audio_native": + from runner.adapters.qwen3_mlx import Qwen3MLXAudioAdapter + + return Qwen3MLXAudioAdapter(profile, **kwargs) raise UnknownModelError( f"No adapter registered for family={profile.family!r}, " diff --git a/runner/adapters/qwen3_mlx.py b/runner/adapters/qwen3_mlx.py new file mode 100644 index 0000000..5708f97 --- /dev/null +++ b/runner/adapters/qwen3_mlx.py @@ -0,0 +1,502 @@ +"""Apple-Silicon backends for Qwen3-TTS MLX model bundles. + +The two registered bundles intentionally use different loaders: + +* ``aufklarer/...-MLX-8bit`` is a Speech Swift/minmax bundle and is invoked + through the ``speech`` CLI that officially supports a full Hugging Face ID. +* ``mlx-community/...-Base-4bit`` is an mlx-audio/affine bundle and is loaded + in-process with ``mlx_audio.tts.utils.load_model``. + +Keeping the adapters separate prevents silent tensor/config incompatibilities. +""" + +from __future__ import annotations + +import importlib.util +import io +import os +import platform +import shutil +import subprocess +import tempfile +import threading +from pathlib import Path +from typing import Any, Callable + +import numpy as np +import scipy.io.wavfile + +from runner.adapters.base import EngineAdapter +from runner.errors import ( + BackendUnavailableError, + CapabilityError, + InvalidRequestForModelError, +) +from runner.model_registry import ModelProfile +from runner.types import SpeechRequest + +_OPENAI_DEFAULT_VOICES = { + "alloy", + "ash", + "ballad", + "cedar", + "coral", + "default", + "echo", + "fable", + "juniper", + "marin", + "nova", + "onyx", + "sage", + "shimmer", + "verse", +} + + +def is_apple_silicon( + system: str | None = None, + machine: str | None = None, +) -> bool: + """Return whether the current process is native Apple Silicon macOS.""" + resolved_system = (system or platform.system()).lower() + resolved_machine = (machine or platform.machine()).lower() + return resolved_system == "darwin" and resolved_machine in {"arm64", "aarch64"} + + +def _resolve_reference_audio(request: SpeechRequest) -> str | None: + """Resolve explicit ``ref_audio`` or a file-valued OpenAI ``voice``.""" + raw = request.ref_audio + if raw is None and request.voice: + candidate = Path(request.voice).expanduser() + if candidate.is_file(): + raw = str(candidate) + elif request.voice.lower() not in _OPENAI_DEFAULT_VOICES: + raise InvalidRequestForModelError( + "This Qwen3-TTS Base profile has no named speaker presets. " + "Use voice='default', omit voice, or pass a local reference-audio " + "path in 'ref_audio' (or 'voice')." + ) + + if raw is None: + return None + + path = Path(raw).expanduser() + if not path.is_file(): + raise InvalidRequestForModelError( + f"Reference audio does not exist or is not a file: {path}" + ) + return str(path.resolve()) + + +def _common_capabilities(profile: ModelProfile, available: bool, backend: str) -> dict[str, Any]: + return { + "model": profile.key, + "family": profile.family, + "backend": backend, + "hf_model_id": profile.hf_model_id, + "quantization": profile.quantization, + "platforms": list(profile.platforms), + "sample_rate": profile.sample_rate, + "supports_stream": profile.supports_stream, + "supports_multispeaker": profile.supports_multispeaker, + "supports_voice_list": profile.supports_voice_list, + "supports_reference_audio": profile.supports_reference_audio, + "status": "available" if available else "backend_unavailable", + } + + +class Qwen3SpeechSwiftAdapter(EngineAdapter): + """Serve the exact aufklarer 8-bit minmax bundle through Speech Swift.""" + + def __init__(self, profile: ModelProfile, **kwargs: Any) -> None: + super().__init__(profile, **kwargs) + self._system = kwargs.get("platform_system") + self._machine = kwargs.get("platform_machine") + self._run_command: Callable[..., Any] = kwargs.get("run_command", subprocess.run) + self._lock = threading.Lock() + + model_override = kwargs.get("model_path") or os.environ.get("QWEN3_TTS_MODEL_ID") + self.model_id = str(model_override) if model_override else profile.hf_model_id + + configured_binary = kwargs.get("speech_bin") or os.environ.get("SPEECH_SWIFT_BIN") + self.binary = self._find_binary(configured_binary) + self._backend_error: str | None = None + + @staticmethod + def _find_binary(configured: str | os.PathLike[str] | None) -> str | None: + if configured: + configured_str = str(configured) + found = shutil.which(configured_str) + if found: + return found + path = Path(configured_str).expanduser() + if path.is_file() and os.access(path, os.X_OK): + return str(path.resolve()) + return None + return shutil.which("speech") or shutil.which("audio") + + def _availability_error(self) -> str | None: + if not is_apple_silicon(self._system, self._machine): + return "Speech Swift requires native Apple Silicon macOS (arm64), not Rosetta/x86_64." + if self.binary is None: + return ( + "Speech Swift was not found. Install native ARM Homebrew and run " + "'brew install speech', or set SPEECH_SWIFT_BIN." + ) + return None + + def is_available(self) -> bool: + self._backend_error = self._availability_error() + return self._backend_error is None + + def capabilities(self) -> dict[str, Any]: + available = self.is_available() + result = _common_capabilities(self.profile, available, "speech_swift_cli") + result["model_source"] = self.model_id + if self.binary: + result["binary"] = self.binary + if self._backend_error: + result["error"] = self._backend_error + return result + + def list_voices(self) -> list[dict[str, Any]]: + return [ + { + "id": "default", + "name": "Default / unconditioned", + "object": "voice", + "category": "qwen3_base", + }, + { + "id": "reference_audio", + "name": "Reference audio path", + "object": "voice", + "category": "voice_cloning", + }, + ] + + def synthesize(self, request: SpeechRequest) -> tuple[bytes, str]: + if not self.is_available(): + raise BackendUnavailableError(self.profile.key, self._backend_error) + if not request.input: + raise InvalidRequestForModelError("Field 'input' is required.") + if request.speakers: + raise InvalidRequestForModelError( + "Qwen3-TTS Base does not accept multi-speaker 'speakers' turns." + ) + if request.stream: + raise CapabilityError(self.profile.key, "stream") + if request.speed not in (None, 1.0): + raise InvalidRequestForModelError( + "Speech Swift's Qwen3 backend does not expose speed control; use speed=1.0." + ) + if request.instruct: + raise InvalidRequestForModelError( + "The selected Base model does not support 'instruct'; use a CustomVoice model." + ) + if request.ref_text: + raise InvalidRequestForModelError( + "The Speech Swift 8-bit CLI does not expose reference transcripts. " + "Use the 4-bit mlx-audio profile for ref_audio + ref_text ICL cloning." + ) + if request.top_p is not None or request.repetition_penalty is not None: + raise InvalidRequestForModelError( + "The Speech Swift 8-bit CLI exposes temperature, top_k, and max_tokens, " + "but not top_p or repetition_penalty." + ) + + ref_audio = _resolve_reference_audio(request) + language = request.language or "english" + temperature = request.temp if request.temp is not None else 0.3 + top_k = request.top_k if request.top_k is not None else 50 + max_tokens = request.max_tokens if request.max_tokens is not None else 500 + + assert self.binary is not None + with tempfile.TemporaryDirectory(prefix="qwen3-tts-") as tmpdir: + output_path = Path(tmpdir) / "speech.wav" + cmd = [ + self.binary, + "speak", + request.input, + "--engine", + "qwen3", + "--output", + str(output_path), + "--language", + language, + "--model", + self.model_id, + "--temperature", + str(temperature), + "--top-k", + str(top_k), + "--max-tokens", + str(max_tokens), + ] + if ref_audio: + cmd.extend(["--voice-sample", ref_audio]) + + timeout = float(os.environ.get("QWEN3_TTS_TIMEOUT_SECONDS", "1800")) + env = os.environ.copy() + env.setdefault("NO_COLOR", "1") + + try: + # Serialise calls to avoid loading multiple 1.7B models into unified + # memory when the HTTP server is hit concurrently. + with self._lock: + completed = self._run_command( + cmd, + check=False, + capture_output=True, + text=True, + timeout=timeout, + env=env, + ) + except subprocess.TimeoutExpired as exc: + raise BackendUnavailableError( + self.profile.key, + f"Speech Swift synthesis exceeded {timeout:g} seconds.", + ) from exc + except OSError as exc: + raise BackendUnavailableError(self.profile.key, str(exc)) from exc + + return_code = int(getattr(completed, "returncode", 1)) + if return_code != 0: + stderr = str(getattr(completed, "stderr", "")).strip() + stdout = str(getattr(completed, "stdout", "")).strip() + detail = stderr or stdout or f"speech exited with status {return_code}" + detail = detail[-2000:] + raise BackendUnavailableError(self.profile.key, detail) + if not output_path.is_file() or output_path.stat().st_size == 0: + raise BackendUnavailableError( + self.profile.key, + "Speech Swift completed without producing a WAV file.", + ) + return output_path.read_bytes(), "audio/wav" + + def stream(self, request: SpeechRequest) -> Any: + raise CapabilityError(self.profile.key, "stream") + + def health(self) -> dict[str, Any]: + available = self.is_available() + result = { + "adapter": "qwen3_speech_swift", + "model": self.profile.key, + "model_source": self.model_id, + "available": available, + "loaded": False, + "platform": f"{platform.system()}-{platform.machine()}", + } + if self.binary: + result["binary"] = self.binary + if self._backend_error: + result["error"] = self._backend_error + return result + + +class Qwen3MLXAudioAdapter(EngineAdapter): + """Serve mlx-community Qwen3-TTS bundles in-process with mlx-audio.""" + + def __init__(self, profile: ModelProfile, **kwargs: Any) -> None: + super().__init__(profile, **kwargs) + self._system = kwargs.get("platform_system") + self._machine = kwargs.get("platform_machine") + self._load_model_fn: Callable[[Any], Any] | None = kwargs.get("load_model_fn") + self._load_audio_fn: Callable[..., Any] | None = kwargs.get("load_audio_fn") + self._model: Any | None = kwargs.get("model") + self._runtime_loaded = self._model is not None + self._backend_error: str | None = None + self._lock = threading.Lock() + self.model_source = self._resolve_model_source(kwargs.get("model_path")) + + def _resolve_model_source(self, override: Any | None) -> str: + if override is not None: + raw = str(override) + path = Path(raw).expanduser() + return str(path.resolve()) if path.exists() else raw + + local = Path(self.profile.default_local_dir).expanduser() + if local.is_dir() and any(local.iterdir()): + return str(local.resolve()) + # mlx-audio accepts either a local path or a Hugging Face repository ID, + # so a fresh Mac can download into its normal HF cache on first load. + return self.profile.hf_model_id + + def _availability_error(self) -> str | None: + if not is_apple_silicon(self._system, self._machine): + return "mlx-audio requires native Apple Silicon macOS (arm64)." + if self._load_model_fn is not None or self._model is not None: + return None + if importlib.util.find_spec("mlx_audio") is None: + return "mlx-audio is not installed. Run 'uv sync --extra mac'." + if importlib.util.find_spec("mlx") is None: + return "The MLX runtime is not installed. Run 'uv sync --extra mac'." + return None + + def is_available(self) -> bool: + self._backend_error = self._availability_error() + return self._backend_error is None + + def _ensure_runtime_loaded(self) -> None: + if self._runtime_loaded: + return + if not self.is_available(): + raise BackendUnavailableError(self.profile.key, self._backend_error) + + # Loading a 1.7B model twice can exhaust unified memory. Re-check state + # under the same lock used for generation before constructing it. + with self._lock: + if self._runtime_loaded: + return + try: + loader = self._load_model_fn + if loader is None: + from mlx_audio.tts.utils import load_model + + loader = load_model + self._model = loader(self.model_source) + self._runtime_loaded = True + self._backend_error = None + except Exception as exc: + self._backend_error = str(exc) + raise BackendUnavailableError(self.profile.key, self._backend_error) from exc + + def _load_reference_audio(self, path: str | None) -> Any | None: + """Load and resample a reference file to the model's native sample rate.""" + if path is None: + return None + + loader = self._load_audio_fn + if loader is None: + # mlx-audio's Qwen generate() API expects an MLX waveform, not a + # filesystem path. Import lazily so registry/API inspection remains + # usable on non-macOS hosts without MLX installed. + from mlx_audio.tts.generate import load_audio + + loader = load_audio + + try: + return loader( + path, + sample_rate=self.profile.sample_rate or 24_000, + ) + except Exception as exc: + raise InvalidRequestForModelError( + f"Failed to load reference audio '{path}': {exc}" + ) from exc + + def capabilities(self) -> dict[str, Any]: + available = self.is_available() + result = _common_capabilities(self.profile, available, "mlx_audio_native") + result["model_source"] = self.model_source + result["loaded"] = self._runtime_loaded + if self._backend_error: + result["error"] = self._backend_error + return result + + def list_voices(self) -> list[dict[str, Any]]: + return [ + { + "id": "default", + "name": "Default / unconditioned", + "object": "voice", + "category": "qwen3_base", + }, + { + "id": "reference_audio", + "name": "Reference audio path", + "object": "voice", + "category": "voice_cloning", + }, + ] + + def synthesize(self, request: SpeechRequest) -> tuple[bytes, str]: + if not request.input: + raise InvalidRequestForModelError("Field 'input' is required.") + if request.speakers: + raise InvalidRequestForModelError( + "Qwen3-TTS Base does not accept multi-speaker 'speakers' turns." + ) + if request.stream: + raise CapabilityError(self.profile.key, "stream") + if request.speed not in (None, 1.0): + raise InvalidRequestForModelError( + "mlx-audio currently does not implement Qwen3-TTS speed control; use speed=1.0." + ) + if request.instruct: + raise InvalidRequestForModelError( + "The selected Base model does not support 'instruct'; use a CustomVoice model." + ) + + ref_audio_path = _resolve_reference_audio(request) + if request.ref_text and ref_audio_path is None: + raise InvalidRequestForModelError( + "Field 'ref_text' requires an actual reference-audio file, not voice='default'." + ) + self._ensure_runtime_loaded() + ref_audio = self._load_reference_audio(ref_audio_path) + assert self._model is not None + + kwargs: dict[str, Any] = { + "text": request.input, + "voice": None, + "temperature": request.temp if request.temp is not None else 0.9, + "speed": 1.0, + "lang_code": request.language or "auto", + "ref_audio": ref_audio, + "ref_text": request.ref_text, + "max_tokens": request.max_tokens if request.max_tokens is not None else 4096, + "verbose": False, + "stream": False, + "top_k": request.top_k if request.top_k is not None else 50, + "top_p": request.top_p if request.top_p is not None else 1.0, + "repetition_penalty": ( + request.repetition_penalty + if request.repetition_penalty is not None + else 1.05 + ), + } + + try: + audio_parts: list[np.ndarray] = [] + sample_rate = self.profile.sample_rate or 24000 + with self._lock: + for result in self._model.generate(**kwargs): + part = np.asarray(result.audio, dtype=np.float32).reshape(-1) + if part.size: + audio_parts.append(part.copy()) + sample_rate = int(getattr(result, "sample_rate", sample_rate)) + except InvalidRequestForModelError: + raise + except Exception as exc: + self._backend_error = str(exc) + raise BackendUnavailableError(self.profile.key, self._backend_error) from exc + + if not audio_parts: + raise BackendUnavailableError( + self.profile.key, + "mlx-audio returned no audio samples.", + ) + + audio = np.concatenate(audio_parts).astype(np.float32, copy=False) + buffer = io.BytesIO() + scipy.io.wavfile.write(buffer, sample_rate, audio) + return buffer.getvalue(), "audio/wav" + + def stream(self, request: SpeechRequest) -> Any: + raise CapabilityError(self.profile.key, "stream") + + def health(self) -> dict[str, Any]: + available = self.is_available() + result = { + "adapter": "qwen3_mlx_audio", + "model": self.profile.key, + "model_source": self.model_source, + "available": available, + "loaded": self._runtime_loaded, + "platform": f"{platform.system()}-{platform.machine()}", + } + if self._backend_error: + result["error"] = self._backend_error + return result diff --git a/runner/api.py b/runner/api.py new file mode 100644 index 0000000..5ebba2d --- /dev/null +++ b/runner/api.py @@ -0,0 +1,316 @@ +"""Adapter-driven FastAPI application for native/non-demo TTS backends.""" + +from __future__ import annotations + +import asyncio +import io +import logging +import os +from typing import Any + +from fastapi import FastAPI, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, Response + +from runner.adapter_factory import make_adapter +from runner.errors import ( + BackendUnavailableError, + CapabilityError, + InvalidRequestForModelError, + UnknownModelError, +) +from runner.model_registry import ( + aliases_for_model, + get_model_profile, + list_profiles, + resolve_model_key, +) +from runner.types import ( + SpeechRequest, + validate_for_longform, + validate_for_qwen3_tts, + validate_for_realtime, +) + +logger = logging.getLogger(__name__) + + +class AudioEncodingError(RuntimeError): + """Raised when WAV output cannot be converted to the requested format.""" + + +def _error_response(message: str, status_code: int, code: str) -> JSONResponse: + return JSONResponse( + status_code=status_code, + content={ + "error": { + "message": message, + "type": "invalid_request_error" if status_code < 500 else "server_error", + "param": None, + "code": code, + } + }, + ) + + +def _normalise_wav_mime(mime_type: str) -> bool: + return mime_type.lower() in {"wav", "audio/wav", "audio/x-wav", "audio/wave"} + + +def _encode_response(audio: bytes, source_mime: str, response_format: str) -> tuple[bytes, str]: + requested = response_format.lower() + if requested not in {"wav", "mp3", "opus"}: + raise InvalidRequestForModelError( + "Unsupported response_format. Use one of: wav, mp3, opus." + ) + + if requested == "wav": + if not _normalise_wav_mime(source_mime): + raise AudioEncodingError( + f"Adapter returned {source_mime!r}; WAV passthrough is unavailable." + ) + return audio, "audio/wav" + + if not _normalise_wav_mime(source_mime): + raise AudioEncodingError( + f"Cannot transcode adapter output {source_mime!r}; expected WAV." + ) + + try: + from pydub import AudioSegment + from pydub.exceptions import CouldntDecodeError, CouldntEncodeError + + segment = AudioSegment.from_file(io.BytesIO(audio), format="wav") + output = io.BytesIO() + if requested == "mp3": + segment.export(output, format="mp3") + return output.getvalue(), "audio/mpeg" + segment.export(output, format="opus", codec="libopus") + return output.getvalue(), "audio/opus" + except (FileNotFoundError, CouldntDecodeError, CouldntEncodeError, OSError) as exc: + raise AudioEncodingError( + "Audio transcoding failed. Install ffmpeg or request response_format='wav'." + ) from exc + + +def _validate_request(profile_family: str, request: SpeechRequest) -> None: + if profile_family == "realtime": + errors = validate_for_realtime(request) + elif profile_family == "longform": + errors = validate_for_longform(request) + elif profile_family == "qwen3_tts": + errors = validate_for_qwen3_tts(request) + else: + errors = [f"Unsupported model family: {profile_family}"] + if errors: + raise InvalidRequestForModelError(" ".join(errors)) + + +def create_app( + *, + model_key: str, + model_path: str | os.PathLike[str] | None = None, + device: str | None = None, + max_concurrent_requests: int = 1, +) -> FastAPI: + """Create a FastAPI app backed by one canonical model adapter.""" + canonical_key = resolve_model_key(model_key, include_native=True) + profile = get_model_profile(canonical_key, include_native=True) + if profile.loader_mode == "subprocess_demo": + raise ValueError( + "Realtime demo profiles must be started through 'vibevoice-server' so " + "their vendored FastAPI/WebSocket application is launched correctly." + ) + + adapter_kwargs: dict[str, Any] = {} + if model_path is not None: + adapter_kwargs["model_path"] = model_path + if device is not None: + adapter_kwargs["device"] = device + adapter = make_adapter(canonical_key, **adapter_kwargs) + + concurrency = max(1, int(max_concurrent_requests)) + semaphore = asyncio.Semaphore(concurrency) + + app = FastAPI( + title="VibeVoice / Qwen3 TTS API", + version="0.2.0", + description="OpenAI-compatible local TTS API backed by the runner adapter registry.", + ) + app.state.model_key = canonical_key + app.state.profile = profile + app.state.adapter = adapter + app.state.semaphore = semaphore + + @app.exception_handler(HTTPException) + async def _http_exception_handler( + _request: Request, exc: HTTPException + ) -> JSONResponse: + return _error_response(str(exc.detail), exc.status_code, "http_error") + + @app.exception_handler(RequestValidationError) + async def _request_validation_handler( + _request: Request, exc: RequestValidationError + ) -> JSONResponse: + return _error_response(str(exc), 422, "request_validation_error") + + @app.exception_handler(UnknownModelError) + async def _unknown_model_handler( + _request: Request, exc: UnknownModelError + ) -> JSONResponse: + return _error_response(str(exc), 400, "unknown_model") + + @app.exception_handler(InvalidRequestForModelError) + async def _invalid_request_handler( + _request: Request, exc: InvalidRequestForModelError + ) -> JSONResponse: + return _error_response(str(exc), 422, "invalid_request_for_model") + + @app.exception_handler(CapabilityError) + async def _capability_handler( + _request: Request, exc: CapabilityError + ) -> JSONResponse: + return _error_response(str(exc), 400, "unsupported_capability") + + @app.exception_handler(BackendUnavailableError) + async def _backend_handler( + _request: Request, exc: BackendUnavailableError + ) -> JSONResponse: + return _error_response(str(exc), 503, "backend_unavailable") + + @app.exception_handler(AudioEncodingError) + async def _encoding_handler( + _request: Request, exc: AudioEncodingError + ) -> JSONResponse: + return _error_response(str(exc), 501, "audio_encoding_unavailable") + + @app.get("/") + async def root() -> dict[str, Any]: + return { + "service": "vibevoice-realtimeFASTAPI", + "active_model": canonical_key, + "docs": "/docs", + "models": "/v1/models", + "speech": "/v1/audio/speech", + } + + @app.get("/health") + async def health() -> JSONResponse: + details = adapter.health() + available = bool(details["available"]) if "available" in details else adapter.is_available() + return JSONResponse( + status_code=200 if available else 503, + content={ + "status": "ok" if available else "degraded", + "active_model": canonical_key, + **details, + }, + ) + + @app.get("/config") + async def config() -> dict[str, Any]: + models = [] + for item in list_profiles(include_native=True): + row = item.as_dict() + row["aliases"] = aliases_for_model(item.key, include_native=True) + row["active"] = item.key == canonical_key + models.append(row) + return { + "active_model": canonical_key, + "active_capabilities": adapter.capabilities(), + "max_concurrent_requests": concurrency, + "available_models": models, + } + + @app.get("/v1/models") + async def models() -> dict[str, Any]: + return { + "object": "list", + "data": [ + { + "id": canonical_key, + "object": "model", + "created": 0, + "owned_by": "local", + "family": profile.family, + "backend": profile.loader_mode, + "hf_model_id": profile.hf_model_id, + "aliases": aliases_for_model(canonical_key, include_native=True), + } + ], + } + + @app.get("/v1/audio/voices") + async def voices() -> dict[str, Any]: + return { + "object": "list", + "model": canonical_key, + "data": adapter.list_voices(), + } + + @app.post("/v1/audio/speech") + async def speech(request: SpeechRequest) -> Response: + requested_key = ( + resolve_model_key(request.model, include_native=True) + if request.model + else canonical_key + ) + if requested_key != canonical_key: + raise HTTPException( + status_code=409, + detail=( + f"This server instance has '{canonical_key}' loaded, but the request " + f"selected '{requested_key}'. Start another instance with " + f"--model {requested_key}." + ), + ) + + request.model = canonical_key + _validate_request(profile.family, request) + if not adapter.is_available(): + health_details = adapter.health() + raise BackendUnavailableError( + canonical_key, + str(health_details.get("error") or "Backend readiness check failed."), + ) + + try: + async with semaphore: + audio, source_mime = await asyncio.to_thread(adapter.synthesize, request) + encoded, media_type = await asyncio.to_thread( + _encode_response, + audio, + source_mime, + request.response_format, + ) + return Response( + content=encoded, + media_type=media_type, + headers={"X-TTS-Model": canonical_key}, + ) + except (BackendUnavailableError, CapabilityError, InvalidRequestForModelError): + raise + except Exception as exc: + logger.exception("Speech synthesis failed for model %s", canonical_key) + raise BackendUnavailableError(canonical_key, str(exc)) from exc + + return app + + +def create_app_from_env() -> FastAPI: + """Uvicorn factory used by ``--reload`` and process managers.""" + model_key = os.environ.get("TTS_MODEL") + if not model_key: + raise RuntimeError( + "TTS_MODEL is required when starting runner.api directly. " + "Use 'vibevoice-server --model ...' for normal launches." + ) + model_path = os.environ.get("MODEL_PATH") or None + device = os.environ.get("MODEL_DEVICE") or None + max_concurrency = int(os.environ.get("MAX_CONCURRENT_REQUESTS", "1")) + return create_app( + model_key=model_key, + model_path=model_path, + device=device, + max_concurrent_requests=max_concurrency, + ) diff --git a/runner/cli.py b/runner/cli.py new file mode 100644 index 0000000..14b730c --- /dev/null +++ b/runner/cli.py @@ -0,0 +1,150 @@ +"""Command-line launcher for all registered TTS backends.""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +from runner.adapter_factory import make_adapter +from runner.api import create_app +from runner.errors import UnknownModelError +from runner.model_registry import ( + aliases_for_model, + get_model_profile, + list_profiles, + resolve_model_key, +) + + +def _print_models() -> None: + print("Registered models:\n") + for profile in list_profiles(include_native=True): + aliases = ", ".join(aliases_for_model(profile.key, include_native=True)) or "-" + print(f" {profile.key:24} {profile.loader_mode:20} {profile.hf_model_id}") + print(f" aliases: {aliases}") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Launch a registered VibeVoice or Qwen3-TTS backend" + ) + parser.add_argument( + "--model", + default=os.environ.get("TTS_MODEL", "realtime-0.5b"), + help="Canonical model key or alias (default: realtime-0.5b)", + ) + parser.add_argument( + "--model-path", + default=None, + help=( + "Override the model source. VibeVoice expects a local path; the Speech Swift " + "8-bit backend expects a Hugging Face model ID; mlx-audio accepts either a " + "local directory or a Hugging Face model ID." + ), + ) + parser.add_argument("--host", default="0.0.0.0") + parser.add_argument("--port", type=int, default=8000) + parser.add_argument( + "--device", + default="auto", + choices=["auto", "cpu", "cuda", "mps", "mlx"], + help="PyTorch device; Qwen3 MLX profiles always use Metal/MLX.", + ) + parser.add_argument("--reload", action="store_true") + parser.add_argument("--inference-steps", type=int, default=5) + parser.add_argument("--lazy-load", action="store_true") + parser.add_argument("--startup-warmup", dest="startup_warmup", action="store_true", default=None) + parser.add_argument("--no-startup-warmup", dest="startup_warmup", action="store_false") + parser.add_argument("--max-concurrent-requests", type=int, default=1) + parser.add_argument("--list-models", action="store_true") + return parser + + +def main(argv: list[str] | None = None) -> None: + args = _parser().parse_args(argv) + if args.list_models: + _print_models() + return + + try: + model_key = resolve_model_key(args.model, include_native=True) + except UnknownModelError as exc: + print(f"Error: {exc}", file=sys.stderr) + raise SystemExit(2) from exc + + profile = get_model_profile(model_key, include_native=True) + device: str | None + if profile.family == "qwen3_tts": + device = "mlx" + elif args.device == "auto": + # Import the PyTorch-backed detector only for VibeVoice profiles. + # A Qwen-only Mac installation deliberately has no torch dependency. + from runner.adapters.realtime_demo import detect_device + + device = detect_device() + else: + device = args.device + + adapter_kwargs: dict[str, object] = {"device": device} + if args.model_path: + adapter_kwargs["model_path"] = ( + args.model_path + if profile.family == "qwen3_tts" + else Path(args.model_path).expanduser() + ) + adapter = make_adapter(model_key, **adapter_kwargs) + + print(f"Model: {model_key}") + print(f"Backend: {profile.loader_mode}") + print(f"Source: {args.model_path or profile.hf_model_id}") + print(f"Device: {device}") + + if profile.loader_mode == "subprocess_demo": + adapter.launch( + project_root=Path(__file__).resolve().parents[1], + host=args.host, + port=args.port, + inference_steps=args.inference_steps, + lazy_load=args.lazy_load, + startup_warmup=args.startup_warmup, + reload=args.reload, + ) + return + + if not adapter.is_available(): + details = adapter.health() + error = details.get("error", "backend readiness check failed") + print(f"Backend unavailable: {error}", file=sys.stderr) + raise SystemExit(1) + + import uvicorn + + if args.reload: + os.environ["TTS_MODEL"] = model_key + if args.model_path: + os.environ["MODEL_PATH"] = args.model_path + else: + os.environ.pop("MODEL_PATH", None) + if device: + os.environ["MODEL_DEVICE"] = device + os.environ["MAX_CONCURRENT_REQUESTS"] = str( + max(1, args.max_concurrent_requests) + ) + uvicorn.run( + "runner.api:create_app_from_env", + factory=True, + host=args.host, + port=args.port, + reload=True, + ) + return + + app = create_app( + model_key=model_key, + model_path=args.model_path, + device=device, + max_concurrent_requests=max(1, args.max_concurrent_requests), + ) + uvicorn.run(app, host=args.host, port=args.port) diff --git a/runner/errors.py b/runner/errors.py index c72b601..58d8ba4 100644 --- a/runner/errors.py +++ b/runner/errors.py @@ -8,7 +8,7 @@ def __init__(self, model_key: str) -> None: self.model_key = model_key super().__init__( f"Unknown model: '{model_key}'. " - "Use GET /config to see available models and aliases." + "Use GET /v1/models or GET /config to see available models and aliases." ) @@ -18,27 +18,22 @@ class CapabilityError(Exception): def __init__(self, model_key: str, capability: str) -> None: self.model_key = model_key self.capability = capability - super().__init__( - f"Model '{model_key}' does not support '{capability}'." - ) + super().__init__(f"Model '{model_key}' does not support '{capability}'.") class BackendUnavailableError(Exception): - """Raised when the backend required by a model is not installed/configured.""" + """Raised when a model's runtime backend is not installed or configured.""" def __init__(self, model_key: str, detail: str | None = None) -> None: self.model_key = model_key - msg = ( - f"Model '{model_key}' is registered but no compatible long-form backend " - "is installed/configured." - ) + msg = f"Model '{model_key}' is registered but its runtime backend is unavailable." if detail: msg += f" {detail}" super().__init__(msg) class InvalidRequestForModelError(Exception): - """Raised when request parameters are incompatible with the chosen model family.""" + """Raised when request parameters are incompatible with a model family.""" def __init__(self, detail: str) -> None: super().__init__(detail) diff --git a/runner/model_registry.py b/runner/model_registry.py index 30ba118..995de5b 100644 --- a/runner/model_registry.py +++ b/runner/model_registry.py @@ -1,32 +1,56 @@ -"""Model registry ā€“ canonical profiles, aliases, and lookup helpers.""" +"""Model registry ā€“ canonical profiles, aliases, and lookup helpers. + +The vendored realtime VibeVoice application predates native adapter serving and +imports these helpers directly. Its default registry view therefore remains +VibeVoice-only. New launchers opt into ``include_native=True`` so Apple MLX +profiles cannot be advertised by, or accidentally routed through, the old +realtime service. +""" from __future__ import annotations import os -from dataclasses import dataclass +from dataclasses import asdict, dataclass from typing import Literal from runner.errors import UnknownModelError +ModelFamily = Literal["realtime", "longform", "qwen3_tts"] +LoaderMode = Literal[ + "subprocess_demo", + "native_longform", + "speech_swift_cli", + "mlx_audio_native", +] + @dataclass(frozen=True) class ModelProfile: - """Describes a supported VibeVoice model variant.""" + """Describe one model and the backend required to serve it.""" key: str hf_model_id: str default_local_dir: str - family: Literal["realtime", "longform"] - loader_mode: Literal["subprocess_demo", "native_longform"] + family: ModelFamily + loader_mode: LoaderMode supports_stream: bool supports_multispeaker: bool supports_voice_list: bool + supports_reference_audio: bool = False + sample_rate: int | None = None + quantization: str | None = None + platforms: tuple[str, ...] = ("linux", "darwin", "windows") + description: str = "" + def as_dict(self) -> dict[str, object]: + """Return a JSON-serialisable representation.""" + value = asdict(self) + value["platforms"] = list(self.platforms) + return value -# --------------------------------------------------------------------------- -# Canonical model profiles -# --------------------------------------------------------------------------- +# Profiles visible to the copied realtime demo. Keep this surface stable so +# the demo cannot claim it loaded an adapter that it never calls. _PROFILES: dict[str, ModelProfile] = { "realtime-0.5b": ModelProfile( key="realtime-0.5b", @@ -37,6 +61,8 @@ class ModelProfile: supports_stream=True, supports_multispeaker=False, supports_voice_list=True, + sample_rate=24000, + description="VibeVoice realtime PyTorch demo backend.", ), "tts-1.5b": ModelProfile( key="tts-1.5b", @@ -47,6 +73,9 @@ class ModelProfile: supports_stream=False, supports_multispeaker=True, supports_voice_list=False, + supports_reference_audio=True, + sample_rate=24000, + description="VibeVoice 1.5B native long-form backend.", ), "tts-7b": ModelProfile( key="tts-7b", @@ -57,68 +86,129 @@ class ModelProfile: supports_stream=False, supports_multispeaker=True, supports_voice_list=False, + supports_reference_audio=True, + sample_rate=24000, + description="VibeVoice 7B native long-form backend.", ), } -# --------------------------------------------------------------------------- -# Alias mapping ā€“ maps convenience / OpenAI-compat names to canonical keys -# --------------------------------------------------------------------------- +# Native adapter profiles are opt-in for registry lookups. This isolates them +# from overrides/app.py, whose implementation always calls the realtime engine. +_NATIVE_PROFILES: dict[str, ModelProfile] = { + "qwen3-tts-mlx-8bit": ModelProfile( + key="qwen3-tts-mlx-8bit", + hf_model_id="aufklarer/Qwen3-TTS-12Hz-1.7B-Base-MLX-8bit", + default_local_dir="models/Qwen3-TTS-12Hz-1.7B-Base-MLX-8bit", + family="qwen3_tts", + loader_mode="speech_swift_cli", + supports_stream=False, + supports_multispeaker=False, + supports_voice_list=True, + supports_reference_audio=True, + sample_rate=24000, + quantization="8bit-minmax", + platforms=("darwin-arm64",), + description=( + "Exact aufklarer 1.7B 8-bit MLX bundle served through Speech Swift " + "on Apple Silicon." + ), + ), + "qwen3-tts-mlx-4bit": ModelProfile( + key="qwen3-tts-mlx-4bit", + hf_model_id="mlx-community/Qwen3-TTS-12Hz-1.7B-Base-4bit", + default_local_dir="models/Qwen3-TTS-12Hz-1.7B-Base-4bit", + family="qwen3_tts", + loader_mode="mlx_audio_native", + supports_stream=False, + supports_multispeaker=False, + supports_voice_list=True, + supports_reference_audio=True, + sample_rate=24000, + quantization="4bit-affine", + platforms=("darwin-arm64",), + description="Qwen3-TTS 1.7B Base 4-bit MLX bundle served with mlx-audio.", + ), +} _ALIASES: dict[str, str] = { - # OpenAI-compat defaults ā†’ realtime for backward compatibility "tts-1": "realtime-0.5b", "tts-1-hd": "realtime-0.5b", - # Friendly long names "vibevoice-realtime-0.5b": "realtime-0.5b", "vibevoice-1.5b": "tts-1.5b", "vibevoice-7b": "tts-7b", } +_NATIVE_ALIASES: dict[str, str] = { + "qwen3-tts-8bit": "qwen3-tts-mlx-8bit", + "qwen3-tts-1.7b-8bit": "qwen3-tts-mlx-8bit", + "qwen3-mlx-8bit": "qwen3-tts-mlx-8bit", + "aufklarer/qwen3-tts-12hz-1.7b-base-mlx-8bit": "qwen3-tts-mlx-8bit", + "qwen3-tts-4bit": "qwen3-tts-mlx-4bit", + "qwen3-tts-1.7b-4bit": "qwen3-tts-mlx-4bit", + "qwen3-mlx-4bit": "qwen3-tts-mlx-4bit", + "mlx-community/qwen3-tts-12hz-1.7b-base-4bit": "qwen3-tts-mlx-4bit", +} + DEFAULT_MODEL_KEY = "realtime-0.5b" -# --------------------------------------------------------------------------- -# Public helpers -# --------------------------------------------------------------------------- +def _profiles(include_native: bool) -> dict[str, ModelProfile]: + if not include_native: + return _PROFILES + return {**_PROFILES, **_NATIVE_PROFILES} + + +def _aliases(include_native: bool) -> dict[str, str]: + if not include_native: + return _ALIASES + return {**_ALIASES, **_NATIVE_ALIASES} + + +def list_model_keys(*, include_native: bool = False) -> list[str]: + """Return canonical model keys for the requested registry surface.""" + return list(_profiles(include_native).keys()) + +def list_profiles(*, include_native: bool = False) -> list[ModelProfile]: + """Return model profiles in registry order.""" + return list(_profiles(include_native).values()) -def list_model_keys() -> list[str]: - """Return all canonical model keys.""" - return list(_PROFILES.keys()) +def list_aliases(*, include_native: bool = False) -> dict[str, str]: + """Return a copy of aliases for the requested registry surface.""" + return dict(_aliases(include_native)) -def list_aliases() -> dict[str, str]: - """Return a copy of the alias mapping.""" - return dict(_ALIASES) +def aliases_for_model(model_key: str, *, include_native: bool = True) -> list[str]: + """Return aliases that resolve to *model_key*.""" + return sorted( + alias for alias, target in _aliases(include_native).items() if target == model_key + ) -def resolve_model_key(requested: str | None) -> str: - """Resolve *requested* (model name, alias, or ``None``) to a canonical key. - Raises :class:`UnknownModelError` if the value cannot be mapped. +def resolve_model_key(requested: str | None, *, include_native: bool = False) -> str: + """Resolve a model name, alias, or ``None`` to a canonical key. + + Native adapter profiles are excluded by default to preserve the contract of + the copied realtime application. The generic launcher and native API pass + ``include_native=True``. """ if requested is None or requested.strip() == "": return DEFAULT_MODEL_KEY normalised = requested.strip().lower() - - # Direct canonical hit - if normalised in _PROFILES: + profiles = _profiles(include_native) + aliases = _aliases(include_native) + if normalised in profiles: return normalised - - # Alias hit - if normalised in _ALIASES: - return _ALIASES[normalised] - + if normalised in aliases: + return aliases[normalised] raise UnknownModelError(normalised) -def get_model_profile(model_key: str) -> ModelProfile: - """Return the :class:`ModelProfile` for a canonical *model_key*. - - Raises :class:`UnknownModelError` if the key is not registered. - """ +def get_model_profile(model_key: str, *, include_native: bool = False) -> ModelProfile: + """Return a profile from the requested registry surface.""" try: - return _PROFILES[model_key] + return _profiles(include_native)[model_key] except KeyError: raise UnknownModelError(model_key) from None diff --git a/runner/types.py b/runner/types.py index ead5fc5..5c7843d 100644 --- a/runner/types.py +++ b/runner/types.py @@ -2,11 +2,11 @@ from __future__ import annotations -from pydantic import BaseModel, field_validator, model_validator +from pydantic import BaseModel, Field, field_validator, model_validator class SpeakerTurn(BaseModel): - """A single speaker turn for multi-speaker dialogue (longform models).""" + """A single speaker turn for multi-speaker dialogue.""" speaker: str text: str @@ -18,35 +18,52 @@ class SpeechRequest(BaseModel): model: str | None = None input: str | None = None voice: str | None = None - response_format: str = "opus" - temp: float | None = None - speed: float | None = None + response_format: str = "wav" + temp: float | None = Field(default=None, ge=0.0, le=2.0) + speed: float | None = Field(default=None, gt=0.0, le=4.0) stream: bool = False speakers: list[SpeakerTurn] | None = None - @field_validator("input", mode="before") + # Qwen3-TTS / voice-cloning controls. Existing clients can ignore these. + language: str | None = None + ref_audio: str | None = None + ref_text: str | None = None + instruct: str | None = None + top_k: int | None = Field(default=None, ge=1, le=1000) + top_p: float | None = Field(default=None, gt=0.0, le=1.0) + max_tokens: int | None = Field(default=None, ge=1, le=8192) + repetition_penalty: float | None = Field(default=None, ge=0.1, le=5.0) + + @field_validator( + "input", + "voice", + "language", + "ref_audio", + "ref_text", + "instruct", + mode="before", + ) @classmethod - def _normalize_empty_input(cls, v: str | None) -> str | None: - if isinstance(v, str) and v.strip() == "": - return None - return v + def _normalise_empty_string(cls, value: str | None) -> str | None: + if isinstance(value, str): + stripped = value.strip() + return stripped or None + return value - @field_validator("voice", mode="before") + @field_validator("response_format", mode="before") @classmethod - def _normalize_empty_voice(cls, v: str | None) -> str | None: - if isinstance(v, str) and v.strip() == "": - return None - return v + def _normalise_response_format(cls, value: str | None) -> str: + return str(value or "wav").strip().lower() @model_validator(mode="after") - def _normalize_empty_speakers(self) -> "SpeechRequest": + def _normalise_empty_speakers(self) -> "SpeechRequest": if self.speakers is not None and len(self.speakers) == 0: self.speakers = None return self def validate_for_realtime(req: SpeechRequest) -> list[str]: - """Return a list of validation errors for a realtime-family request.""" + """Return validation errors for a realtime-family request.""" errors: list[str] = [] if not req.input: errors.append("Field 'input' is required for realtime models.") @@ -56,10 +73,26 @@ def validate_for_realtime(req: SpeechRequest) -> list[str]: def validate_for_longform(req: SpeechRequest) -> list[str]: - """Return a list of validation errors for a longform-family request.""" + """Return validation errors for a longform-family request.""" errors: list[str] = [] if not req.input and not req.speakers: errors.append("Either 'input' or 'speakers' is required for longform models.") if req.stream: errors.append("Streaming is not supported by longform models.") return errors + + +def validate_for_qwen3_tts(req: SpeechRequest) -> list[str]: + """Return validation errors shared by Qwen3-TTS MLX backends.""" + errors: list[str] = [] + if not req.input: + errors.append("Field 'input' is required for Qwen3-TTS models.") + if req.speakers: + errors.append("Field 'speakers' is not supported by Qwen3-TTS Base models.") + if req.stream: + errors.append("Streaming is not exposed by the Qwen3-TTS HTTP adapter yet.") + if req.speed not in (None, 1.0): + errors.append("The Qwen3-TTS backends currently require speed=1.0.") + if req.ref_text and not (req.ref_audio or req.voice): + errors.append("Field 'ref_text' requires reference audio in 'ref_audio' or 'voice'.") + return errors diff --git a/scripts/bootstrap_macos.sh b/scripts/bootstrap_macos.sh new file mode 100644 index 0000000..8674ea3 --- /dev/null +++ b/scripts/bootstrap_macos.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Native Apple-Silicon setup for the Qwen3-TTS MLX profiles. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$PROJECT_ROOT" + +if [[ "$(uname -s)" != "Darwin" ]]; then + echo "Error: this bootstrap is for macOS." >&2 + exit 1 +fi + +if [[ "$(uname -m)" != "arm64" ]]; then + echo "Error: run a native arm64 shell on Apple Silicon; Rosetta/x86_64 is unsupported." >&2 + exit 1 +fi + +if ! command -v uv >/dev/null 2>&1; then + echo "Error: uv is not installed. Install it with:" >&2 + echo " curl -LsSf https://astral.sh/uv/install.sh | sh" >&2 + exit 1 +fi + +# Speech Swift's Homebrew bottle is native-arm64 only. +if [[ ! -x /opt/homebrew/bin/brew ]]; then + echo "Error: native ARM Homebrew was not found at /opt/homebrew/bin/brew." >&2 + echo "Install Homebrew natively, then run this script again." >&2 + exit 1 +fi + +export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" + +echo "Ensuring Python 3.12 and the MLX Python environment..." +uv python install 3.12 +uv sync --python 3.12 --extra mac + +if ! command -v speech >/dev/null 2>&1 && ! command -v audio >/dev/null 2>&1; then + echo "Installing Speech Swift for the exact aufklarer 8-bit model..." + brew install speech +else + echo "Speech Swift is already installed." +fi + +if ! command -v ffmpeg >/dev/null 2>&1; then + echo "Installing ffmpeg for optional MP3/Opus responses..." + brew install ffmpeg +fi + +cat <<'EOF' + +Apple-Silicon setup complete. + +Exact aufklarer 8-bit backend (Speech Swift): + uv run vibevoice-server --model qwen3-tts-mlx-8bit --host 127.0.0.1 --port 8000 + +1.7B 4-bit backend (mlx-audio): + uv run vibevoice-server --model qwen3-tts-mlx-4bit --host 127.0.0.1 --port 8000 + +The first request downloads the selected model to the normal Hugging Face cache. +EOF diff --git a/scripts/bootstrap_uv.sh b/scripts/bootstrap_uv.sh old mode 100755 new mode 100644 index ad1159a..2076364 --- a/scripts/bootstrap_uv.sh +++ b/scripts/bootstrap_uv.sh @@ -1,57 +1,50 @@ -#!/bin/bash -# Bootstrap script to set up VibeVoice locally using uv +#!/usr/bin/env bash +# Bootstrap the PyTorch/VibeVoice runtime with uv. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" - cd "$PROJECT_ROOT" -echo "šŸ”§ Bootstrapping VibeVoice with uv..." +echo "Bootstrapping VibeVoice with uv..." -# Check if uv is installed -if ! command -v uv &> /dev/null; then - echo "āŒ Error: uv is not installed. Please install it first:" - echo " curl -LsSf https://astral.sh/uv/install.sh | sh" +if ! command -v uv >/dev/null 2>&1; then + echo "Error: uv is not installed. Install it first:" >&2 + echo " curl -LsSf https://astral.sh/uv/install.sh | sh" >&2 exit 1 fi -# Ensure Python 3.11 is available + pinned for this repo -echo "šŸ“¦ Ensuring Python 3.11..." uv python install 3.11 >/dev/null 2>&1 || true uv python pin 3.11 -# Create venv if it doesn't exist -if [ ! -d ".venv" ]; then - echo "šŸ Creating virtual environment..." +if [[ ! -d .venv ]]; then uv venv --python 3.11 -else - echo "āœ… Virtual environment already exists" fi -# Clone VibeVoice if it doesn't exist VIBEVOICE_DIR="third_party/VibeVoice" -if [ ! -d "$VIBEVOICE_DIR" ]; then - echo "šŸ“„ Cloning VibeVoice repository..." +if [[ ! -f "$VIBEVOICE_DIR/pyproject.toml" ]]; then mkdir -p third_party - git clone --quiet --branch main --depth 1 \ - https://github.com/microsoft/VibeVoice.git "$VIBEVOICE_DIR" -else - echo "āœ… VibeVoice repository already exists" + if [[ -f .gitmodules ]] && git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "Initializing the VibeVoice submodule..." + git submodule update --init --recursive third_party/VibeVoice + else + echo "Cloning VibeVoice..." + rm -rf "$VIBEVOICE_DIR" + git clone --quiet --branch main --depth 1 \ + https://github.com/microsoft/VibeVoice.git "$VIBEVOICE_DIR" + fi fi -# Apply overrides -if [ -f "overrides/app.py" ]; then - echo "šŸ”§ Applying custom app.py override..." +if [[ -f overrides/app.py ]]; then cp overrides/app.py "$VIBEVOICE_DIR/demo/web/app.py" fi -echo "šŸ“š Syncing dependencies (this will install VibeVoice from third_party/ via pyproject.toml)..." -uv sync +echo "Syncing the VibeVoice dependency set..." +uv sync --extra vibevoice -echo "āœ… Bootstrap complete!" -echo "" -echo "Next steps:" -echo " 1. Download the model: uv run python scripts/download_model.py" -echo " 2. Run the demo: uv run python scripts/run_realtime_demo.py --port 8000" +echo "Installing the populated VibeVoice checkout editable..." +uv pip install -e "$VIBEVOICE_DIR" +echo "Bootstrap complete." +echo "Download: uv run python scripts/download_model.py --model realtime-0.5b" +echo "Run: uv run vibevoice-server --model realtime-0.5b --port 8000" diff --git a/scripts/run_server.py b/scripts/run_server.py index a4688a3..c9c652d 100644 --- a/scripts/run_server.py +++ b/scripts/run_server.py @@ -1,149 +1,16 @@ #!/usr/bin/env python3 -"""Generic multi-model launcher for the VibeVoice TTS runner. +"""Source-tree wrapper for :mod:`runner.cli`.""" -Usage examples:: +from __future__ import annotations - uv run python scripts/run_server.py --model realtime-0.5b --port 8000 - uv run python scripts/run_server.py --model tts-1.5b --port 8000 - -For the realtime adapter this behaves identically to the legacy -``scripts/run_realtime_demo.py`` script. -""" - -import argparse import sys from pathlib import Path -# Ensure project root is on sys.path so ``runner`` is importable. _PROJECT_ROOT = Path(__file__).resolve().parent.parent if str(_PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(_PROJECT_ROOT)) -from runner.adapter_factory import make_adapter # noqa: E402 -from runner.adapters.longform_native import LongformNativeAdapter # noqa: E402 -from runner.adapters.realtime_demo import RealtimeDemoAdapter, detect_device # noqa: E402 -from runner.errors import UnknownModelError # noqa: E402 -from runner.model_registry import get_model_profile, resolve_model_key # noqa: E402 - - -def main() -> None: - parser = argparse.ArgumentParser(description="Generic VibeVoice TTS server launcher") - parser.add_argument( - "--model", - type=str, - default="realtime-0.5b", - help="Model key or alias (default: realtime-0.5b)", - ) - parser.add_argument( - "--model-path", - type=str, - default=None, - help="Override path to the model directory", - ) - parser.add_argument( - "--host", - type=str, - default="0.0.0.0", - help="Host/interface to bind (default: 0.0.0.0)", - ) - parser.add_argument( - "--port", - type=int, - default=8000, - help="Port to run the server on (default: 8000)", - ) - parser.add_argument( - "--device", - type=str, - default=None, - choices=["cpu", "cuda", "mps"], - help="Device (default: auto-detect)", - ) - parser.add_argument( - "--reload", - action="store_true", - help="Enable auto-reload (development)", - ) - parser.add_argument( - "--inference-steps", - type=int, - default=5, - help="Number of inference steps (default: 5)", - ) - parser.add_argument( - "--lazy-load", - action="store_true", - help="Defer model initialization until first request", - ) - parser.add_argument( - "--startup-warmup", - dest="startup_warmup", - action="store_true", - default=None, - help="Warm the model during startup", - ) - parser.add_argument( - "--no-startup-warmup", - dest="startup_warmup", - action="store_false", - help="Skip startup warmup", - ) - args = parser.parse_args() - - # Resolve model - try: - model_key = resolve_model_key(args.model) - except UnknownModelError as exc: - print(f"āŒ {exc}") - sys.exit(1) - - profile = get_model_profile(model_key) - print(f"šŸ“¦ Model: {model_key} (family={profile.family})") - - # Device - device = args.device - if device is None: - device = detect_device() - print(f"šŸ” Auto-detected device: {device}") - - # Model path - model_path = Path(args.model_path) if args.model_path else Path(profile.default_local_dir) - - # Build adapter - adapter = make_adapter( - model_key, - model_path=model_path, - device=device, - ) - - # --- Realtime adapter: launch via existing subprocess demo --- - if isinstance(adapter, RealtimeDemoAdapter): - adapter.launch( - project_root=_PROJECT_ROOT, - host=args.host, - port=args.port, - inference_steps=args.inference_steps, - lazy_load=args.lazy_load, - startup_warmup=args.startup_warmup, - reload=args.reload, - ) - return - - # --- Longform adapter: fail early if backend is unavailable --- - if isinstance(adapter, LongformNativeAdapter): - if not adapter.is_available(): - print(f"āŒ {profile.key} backend is not available.") - print(f" {adapter.get_backend_error()}") - print("\nšŸ’” A compatible long-form backend must be installed.") - print(" See README for details.") - sys.exit(1) - - # TODO: launch longform-compatible serving path when implemented - print(f"šŸš§ Long-form serving for {model_key} is not yet implemented.") - sys.exit(1) - - print(f"āŒ No launch logic for adapter type: {type(adapter).__name__}") - sys.exit(1) +from runner.cli import main # noqa: E402 if __name__ == "__main__": diff --git a/test_macos_mlx.py b/test_macos_mlx.py new file mode 100644 index 0000000..8ee9b94 --- /dev/null +++ b/test_macos_mlx.py @@ -0,0 +1,369 @@ +"""Hardware-free coverage for the Apple-Silicon Qwen3-TTS integration.""" + +from __future__ import annotations + +import sys +import wave +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import numpy as np +from fastapi.testclient import TestClient + +from runner.adapters.base import EngineAdapter +from runner.adapters.qwen3_mlx import ( + Qwen3MLXAudioAdapter, + Qwen3SpeechSwiftAdapter, + is_apple_silicon, +) +from runner.api import create_app +from runner.model_registry import ( + get_model_profile, + list_model_keys, + resolve_model_key, +) +from runner.errors import InvalidRequestForModelError +from runner.types import SpeechRequest, validate_for_qwen3_tts + + +def _write_test_wav(path: Path) -> None: + with wave.open(str(path), "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(24_000) + wav.writeframes(b"\x00\x00" * 32) + + +def test_native_profiles_are_isolated_from_realtime_registry() -> None: + assert "qwen3-tts-mlx-8bit" not in list_model_keys() + assert "qwen3-tts-mlx-8bit" in list_model_keys(include_native=True) + assert resolve_model_key("qwen3-tts-8bit", include_native=True) == "qwen3-tts-mlx-8bit" + + +def test_qwen_profiles_have_exact_sources() -> None: + eight = get_model_profile("qwen3-tts-mlx-8bit", include_native=True) + four = get_model_profile("qwen3-tts-mlx-4bit", include_native=True) + assert eight.hf_model_id == "aufklarer/Qwen3-TTS-12Hz-1.7B-Base-MLX-8bit" + assert eight.loader_mode == "speech_swift_cli" + assert four.hf_model_id == "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-4bit" + assert four.loader_mode == "mlx_audio_native" + + +def test_apple_silicon_detection() -> None: + assert is_apple_silicon("Darwin", "arm64") is True + assert is_apple_silicon("Darwin", "x86_64") is False + assert is_apple_silicon("Linux", "aarch64") is False + + +def test_qwen_validation_accepts_voice_path_as_reference(tmp_path: Path) -> None: + reference = tmp_path / "ref.wav" + _write_test_wav(reference) + request = SpeechRequest( + input="hello", + voice=str(reference), + ref_text="reference transcript", + ) + assert validate_for_qwen3_tts(request) == [] + + +def test_speech_swift_adapter_builds_exact_model_command(tmp_path: Path) -> None: + profile = get_model_profile("qwen3-tts-mlx-8bit", include_native=True) + captured: dict[str, Any] = {} + + def fake_run(command: list[str], **kwargs: Any) -> SimpleNamespace: + captured["command"] = command + captured["kwargs"] = kwargs + output_path = Path(command[command.index("--output") + 1]) + _write_test_wav(output_path) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + adapter = Qwen3SpeechSwiftAdapter( + profile, + platform_system="Darwin", + platform_machine="arm64", + speech_bin="/bin/echo", + run_command=fake_run, + ) + audio, mime = adapter.synthesize( + SpeechRequest( + input="hello from the exact eight bit model", + response_format="wav", + language="english", + ) + ) + + command = captured["command"] + assert command[0] == "/bin/echo" + assert command[:2] == ["/bin/echo", "speak"] + assert command[command.index("--model") + 1] == profile.hf_model_id + assert audio.startswith(b"RIFF") + assert mime == "audio/wav" + + +def test_speech_swift_adapter_rejects_unsupported_icl_transcript(tmp_path: Path) -> None: + profile = get_model_profile("qwen3-tts-mlx-8bit", include_native=True) + reference = tmp_path / "ref.wav" + _write_test_wav(reference) + adapter = Qwen3SpeechSwiftAdapter( + profile, + platform_system="Darwin", + platform_machine="arm64", + speech_bin="/bin/echo", + ) + request = SpeechRequest( + input="hello", + ref_audio=str(reference), + ref_text="reference transcript", + ) + try: + adapter.synthesize(request) + except InvalidRequestForModelError as exc: + assert "4-bit mlx-audio" in str(exc) + else: + raise AssertionError("expected an explicit unsupported-control error") + + +def test_speech_swift_adapter_rejects_non_apple_platform() -> None: + profile = get_model_profile("qwen3-tts-mlx-8bit", include_native=True) + adapter = Qwen3SpeechSwiftAdapter( + profile, + platform_system="Linux", + platform_machine="x86_64", + speech_bin="/bin/echo", + ) + assert adapter.is_available() is False + assert "Apple Silicon" in adapter.health()["error"] + + +def test_mlx_audio_source_falls_back_to_hf_id(tmp_path: Path) -> None: + profile = replace( + get_model_profile("qwen3-tts-mlx-4bit", include_native=True), + default_local_dir=str(tmp_path / "not-downloaded"), + ) + adapter = Qwen3MLXAudioAdapter( + profile, + platform_system="Darwin", + platform_machine="arm64", + model=object(), + ) + assert adapter.model_source == profile.hf_model_id + + +def test_mlx_audio_loader_is_cached() -> None: + profile = get_model_profile("qwen3-tts-mlx-4bit", include_native=True) + calls: list[str] = [] + + class FakeModel: + def generate(self, **_kwargs: Any): + yield SimpleNamespace( + audio=np.zeros(32, dtype=np.float32), + sample_rate=24_000, + ) + + def fake_loader(source: str) -> FakeModel: + calls.append(source) + return FakeModel() + + adapter = Qwen3MLXAudioAdapter( + profile, + platform_system="Darwin", + platform_machine="arm64", + load_model_fn=fake_loader, + ) + request = SpeechRequest(input="hello", response_format="wav") + adapter.synthesize(request) + adapter.synthesize(request) + assert calls == [profile.hf_model_id] + + +def test_mlx_audio_reference_path_is_loaded_as_waveform(tmp_path: Path) -> None: + profile = get_model_profile("qwen3-tts-mlx-4bit", include_native=True) + reference = tmp_path / "reference.wav" + _write_test_wav(reference) + captured: dict[str, Any] = {} + waveform = np.linspace(-0.2, 0.2, 64, dtype=np.float32) + + def fake_load_audio(path: str, *, sample_rate: int) -> np.ndarray: + captured["path"] = path + captured["sample_rate"] = sample_rate + return waveform + + class FakeModel: + def generate(self, **kwargs: Any): + captured["ref_audio"] = kwargs["ref_audio"] + captured["ref_text"] = kwargs["ref_text"] + yield SimpleNamespace( + audio=np.zeros(32, dtype=np.float32), + sample_rate=24_000, + ) + + adapter = Qwen3MLXAudioAdapter( + profile, + platform_system="Darwin", + platform_machine="arm64", + model=FakeModel(), + load_audio_fn=fake_load_audio, + ) + adapter.synthesize( + SpeechRequest( + input="voice clone", + ref_audio=str(reference), + ref_text="reference transcript", + response_format="wav", + ) + ) + + assert captured["path"] == str(reference.resolve()) + assert captured["sample_rate"] == 24_000 + assert captured["ref_audio"] is waveform + assert captured["ref_text"] == "reference transcript" + + +def test_mlx_audio_ref_text_rejects_default_voice_without_audio() -> None: + profile = get_model_profile("qwen3-tts-mlx-4bit", include_native=True) + + class FakeModel: + def generate(self, **_kwargs: Any): + raise AssertionError("generation must not start") + + adapter = Qwen3MLXAudioAdapter( + profile, + platform_system="Darwin", + platform_machine="arm64", + model=FakeModel(), + ) + + try: + adapter.synthesize( + SpeechRequest( + input="hello", + voice="default", + ref_text="orphan transcript", + ) + ) + except InvalidRequestForModelError as exc: + assert "actual reference-audio file" in str(exc) + else: + raise AssertionError("expected ref_text without reference audio to be rejected") + + +def test_mlx_audio_adapter_writes_wav() -> None: + profile = get_model_profile("qwen3-tts-mlx-4bit", include_native=True) + + class FakeModel: + def generate(self, **kwargs: Any): + assert kwargs["text"] == "hello" + assert kwargs["lang_code"] == "english" + yield SimpleNamespace( + audio=np.linspace(-0.1, 0.1, 240, dtype=np.float32), + sample_rate=24_000, + ) + + adapter = Qwen3MLXAudioAdapter( + profile, + platform_system="Darwin", + platform_machine="arm64", + model=FakeModel(), + ) + audio, mime = adapter.synthesize( + SpeechRequest(input="hello", language="english", response_format="wav") + ) + assert audio.startswith(b"RIFF") + assert b"WAVE" in audio[:16] + assert mime == "audio/wav" + + +def test_native_api_routes_to_active_adapter(monkeypatch) -> None: + profile = get_model_profile("qwen3-tts-mlx-4bit", include_native=True) + + class FakeAdapter(EngineAdapter): + def is_available(self) -> bool: + return True + + def capabilities(self) -> dict[str, Any]: + return {"available": True, "model": self.profile.key} + + def list_voices(self) -> list[dict[str, Any]]: + return [{"id": "default", "object": "voice"}] + + def synthesize(self, request: SpeechRequest) -> tuple[bytes, str]: + assert request.model == profile.key + return b"RIFFfakeWAVE", "audio/wav" + + def stream(self, request: SpeechRequest) -> Any: + raise AssertionError("not used") + + def health(self) -> dict[str, Any]: + return {"available": True, "loaded": True} + + monkeypatch.setattr("runner.api.make_adapter", lambda *_args, **_kwargs: FakeAdapter(profile)) + client = TestClient(create_app(model_key=profile.key)) + + response = client.post( + "/v1/audio/speech", + json={ + "model": "qwen3-tts-4bit", + "input": "hello", + "response_format": "wav", + }, + ) + assert response.status_code == 200 + assert response.headers["content-type"].startswith("audio/wav") + assert response.headers["x-tts-model"] == profile.key + + wrong_model = client.post( + "/v1/audio/speech", + json={ + "model": "qwen3-tts-8bit", + "input": "hello", + "response_format": "wav", + }, + ) + assert wrong_model.status_code == 409 + assert "Start another instance" in wrong_model.json()["error"]["message"] + + +def test_native_api_refuses_realtime_demo_profile() -> None: + try: + create_app(model_key="realtime-0.5b") + except ValueError as exc: + assert "vibevoice-server" in str(exc) + else: + raise AssertionError("native API must not wrap the realtime demo adapter") + + +def test_pyproject_keeps_mac_and_vibevoice_stacks_isolated() -> None: + import tomllib + + data = tomllib.loads(Path("pyproject.toml").read_text()) + core = data["project"]["dependencies"] + extras = data["project"]["optional-dependencies"] + assert not any(item.startswith("huggingface-hub") for item in core) + assert any(item.startswith("huggingface-hub>=0.30.0,<1.0") for item in extras["vibevoice"]) + assert any(item.startswith("huggingface-hub>=1.3.0,<2.0") for item in extras["mac"]) + assert any(item.startswith("mlx-audio==0.3.0") for item in extras["mac"]) + assert "sources" not in data.get("tool", {}).get("uv", {}) + assert [ + {"extra": "vibevoice"}, + {"extra": "mac"}, + ] in data["tool"]["uv"]["conflicts"] + + +def test_cli_import_does_not_eagerly_import_realtime_torch_backend() -> None: + sys.modules.pop("runner.adapters.realtime_demo", None) + sys.modules.pop("runner.cli", None) + + import runner.cli # noqa: F401 + + assert "runner.adapters.realtime_demo" not in sys.modules + + +def test_adapter_factory_does_not_eagerly_import_torch_backend() -> None: + # This is the packaging invariant that lets a Qwen-only Mac install omit + # PyTorch/VibeVoice. Importing the factory alone must not load longform code. + sys.modules.pop("runner.adapters.longform_native", None) + import runner.adapter_factory # noqa: F401 + + assert "runner.adapters.longform_native" not in sys.modules