add VibeVoice-Realtime

demo/realtime_model_inference_from_file.py

@@ -0,0 +1,314 @@
+import argparse
+import os
+import re
+import traceback
+from typing import List, Tuple, Union, Dict, Any
+import time
+import torch
+import copy
+
+from vibevoice.modular.modeling_vibevoice_streaming_inference import VibeVoiceStreamingForConditionalGenerationInference
+from vibevoice.processor.vibevoice_streaming_processor import VibeVoiceStreamingProcessor
+from transformers.utils import logging
+
+logging.set_verbosity_info()
+logger = logging.get_logger(__name__)
+
+
+class VoiceMapper:
+    """Maps speaker names to voice file paths"""
+    
+    def __init__(self):
+        self.setup_voice_presets()
+
+        # change name according to our preset voice file
+        new_dict = {}
+        for name, path in self.voice_presets.items():
+            
+            if '_' in name:
+                name = name.split('_')[0]
+            
+            if '-' in name:
+                name = name.split('-')[-1]
+
+            new_dict[name] = path
+        self.voice_presets.update(new_dict)
+        # print(list(self.voice_presets.keys()))
+
+    def setup_voice_presets(self):
+        """Setup voice presets by scanning the voices directory."""
+        voices_dir = os.path.join(os.path.dirname(__file__), "voices/streaming_model")
+        
+        # Check if voices directory exists
+        if not os.path.exists(voices_dir):
+            print(f"Warning: Voices directory not found at {voices_dir}")
+            self.voice_presets = {}
+            self.available_voices = {}
+            return
+        
+        # Scan for all VOICE files in the voices directory
+        self.voice_presets = {}
+        
+        # Get all .pt files in the voices directory
+        pt_files = [f for f in os.listdir(voices_dir) 
+                    if f.lower().endswith('.pt') and os.path.isfile(os.path.join(voices_dir, f))]
+        
+        # Create dictionary with filename (without extension) as key
+        for pt_file in pt_files:
+            # Remove .pt extension to get the name
+            name = os.path.splitext(pt_file)[0]
+            # Create full path
+            full_path = os.path.join(voices_dir, pt_file)
+            self.voice_presets[name] = full_path
+        
+        # Sort the voice presets alphabetically by name for better UI
+        self.voice_presets = dict(sorted(self.voice_presets.items()))
+        
+        # Filter out voices that don't exist (this is now redundant but kept for safety)
+        self.available_voices = {
+            name: path for name, path in self.voice_presets.items()
+            if os.path.exists(path)
+        }
+        
+        print(f"Found {len(self.available_voices)} voice files in {voices_dir}")
+        print(f"Available voices: {', '.join(self.available_voices.keys())}")
+
+    def get_voice_path(self, speaker_name: str) -> str:
+        """Get voice file path for a given speaker name"""
+        # First try exact match
+        if speaker_name in self.voice_presets:
+            return self.voice_presets[speaker_name]
+        
+        # Try partial matching (case insensitive)
+        speaker_lower = speaker_name.lower()
+        for preset_name, path in self.voice_presets.items():
+            if preset_name.lower() in speaker_lower or speaker_lower in preset_name.lower():
+                return path
+        
+        # Default to first voice if no match found
+        default_voice = list(self.voice_presets.values())[0]
+        print(f"Warning: No voice preset found for '{speaker_name}', using default voice: {default_voice}")
+        return default_voice
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="VibeVoiceStreaming Processor TXT Input Test")
+    parser.add_argument(
+        "--model_path",
+        type=str,
+        default="microsoft/VibeVoice-Realtime-0.5B",
+        help="Path to the HuggingFace model directory",
+    )
+    parser.add_argument(
+        "--txt_path",
+        type=str,
+        default="demo/text_examples/1p_vibevoice.txt",
+        help="Path to the txt file containing the script",
+    )
+    parser.add_argument(
+        "--speaker_name",
+        type=str,
+        default="Wayne",
+        help="Single speaker name (e.g., --speaker_name Wayne)",
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="./outputs",
+        help="Directory to save output audio files",
+    )
+    parser.add_argument(
+        "--device",
+        type=str,
+        default=("cuda" if torch.cuda.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu")),
+        help="Device for inference: cuda | mps | cpu",
+    )
+    parser.add_argument(
+        "--cfg_scale",
+        type=float,
+        default=1.5,
+        help="CFG (Classifier-Free Guidance) scale for generation (default: 1.5)",
+    )
+    
+    return parser.parse_args()
+
+def main():
+    args = parse_args()
+
+    # Normalize potential 'mpx' typo to 'mps'
+    if args.device.lower() == "mpx":
+        print("Note: device 'mpx' detected, treating it as 'mps'.")
+        args.device = "mps"
+
+    # Validate mps availability if requested
+    if args.device == "mps" and not torch.backends.mps.is_available():
+        print("Warning: MPS not available. Falling back to CPU.")
+        args.device = "cpu"
+
+    print(f"Using device: {args.device}")
+
+    # Initialize voice mapper
+    voice_mapper = VoiceMapper()
+    
+    # Check if txt file exists
+    if not os.path.exists(args.txt_path):
+        print(f"Error: txt file not found: {args.txt_path}")
+        return
+    
+    # Read and parse txt file
+    print(f"Reading script from: {args.txt_path}")
+    with open(args.txt_path, 'r', encoding='utf-8') as f:
+        scripts = f.read().strip()
+    
+    if not scripts:
+        print("Error: No valid scripts found in the txt file")
+        return
+
+    full_script = scripts.replace("’", "'").replace('“', '"').replace('”', '"')
+
+    print(f"Loading processor & model from {args.model_path}")
+    processor = VibeVoiceStreamingProcessor.from_pretrained(args.model_path)
+
+    # Decide dtype & attention implementation
+    if args.device == "mps":
+        load_dtype = torch.float32  # MPS requires float32
+        attn_impl_primary = "sdpa"  # flash_attention_2 not supported on MPS
+    elif args.device == "cuda":
+        load_dtype = torch.bfloat16
+        attn_impl_primary = "flash_attention_2"
+    else:  # cpu
+        load_dtype = torch.float32
+        attn_impl_primary = "sdpa"
+    print(f"Using device: {args.device}, torch_dtype: {load_dtype}, attn_implementation: {attn_impl_primary}")
+    # Load model with device-specific logic
+    try:
+        if args.device == "mps":
+            model = VibeVoiceStreamingForConditionalGenerationInference.from_pretrained(
+                args.model_path,
+                torch_dtype=load_dtype,
+                attn_implementation=attn_impl_primary,
+                device_map=None,  # load then move
+            )
+            model.to("mps")
+        elif args.device == "cuda":
+            model = VibeVoiceStreamingForConditionalGenerationInference.from_pretrained(
+                args.model_path,
+                torch_dtype=load_dtype,
+                device_map="cuda",
+                attn_implementation=attn_impl_primary,
+            )
+        else:  # cpu
+            model = VibeVoiceStreamingForConditionalGenerationInference.from_pretrained(
+                args.model_path,
+                torch_dtype=load_dtype,
+                device_map="cpu",
+                attn_implementation=attn_impl_primary,
+            )
+    except Exception as e:
+        if attn_impl_primary == 'flash_attention_2':
+            print(f"[ERROR] : {type(e).__name__}: {e}")
+            print(traceback.format_exc())
+            print("Error loading the model. Trying to use SDPA. However, note that only flash_attention_2 has been fully tested, and using SDPA may result in lower audio quality.")
+            model = VibeVoiceStreamingForConditionalGenerationInference.from_pretrained(
+                args.model_path,
+                torch_dtype=load_dtype,
+                device_map=(args.device if args.device in ("cuda", "cpu") else None),
+                attn_implementation='sdpa'
+            )
+            if args.device == "mps":
+                model.to("mps")
+        else:
+            raise e
+
+
+    model.eval()
+    model.set_ddpm_inference_steps(num_steps=5)
+
+    if hasattr(model.model, 'language_model'):
+       print(f"Language model attention: {model.model.language_model.config._attn_implementation}")
+    
+    target_device = args.device if args.device != "cpu" else "cpu"
+    voice_sample = voice_mapper.get_voice_path(args.speaker_name)
+    all_prefilled_outputs = torch.load(voice_sample, map_location=target_device, weights_only=False)
+
+    # Prepare inputs for the model
+    inputs = processor.process_input_with_cached_prompt(
+        text=full_script,
+        cached_prompt=all_prefilled_outputs,
+        padding=True,
+        return_tensors="pt",
+        return_attention_mask=True,
+    )
+
+    # Move tensors to target device
+    for k, v in inputs.items():
+        if torch.is_tensor(v):
+            inputs[k] = v.to(target_device)
+
+    print(f"Starting generation with cfg_scale: {args.cfg_scale}")
+
+    # Generate audio
+    start_time = time.time()
+    outputs = model.generate(
+        **inputs,
+        max_new_tokens=None,
+        cfg_scale=args.cfg_scale,
+        tokenizer=processor.tokenizer,
+        generation_config={'do_sample': False},
+        verbose=True,
+        all_prefilled_outputs=copy.deepcopy(all_prefilled_outputs) if all_prefilled_outputs is not None else None,
+    )
+    generation_time = time.time() - start_time
+    print(f"Generation time: {generation_time:.2f} seconds")
+    
+    # Calculate audio duration and additional metrics
+    if outputs.speech_outputs and outputs.speech_outputs[0] is not None:
+        # Assuming 24kHz sample rate (common for speech synthesis)
+        sample_rate = 24000
+        audio_samples = outputs.speech_outputs[0].shape[-1] if len(outputs.speech_outputs[0].shape) > 0 else len(outputs.speech_outputs[0])
+        audio_duration = audio_samples / sample_rate
+        rtf = generation_time / audio_duration if audio_duration > 0 else float('inf')
+        
+        print(f"Generated audio duration: {audio_duration:.2f} seconds")
+        print(f"RTF (Real Time Factor): {rtf:.2f}x")
+    else:
+        print("No audio output generated")
+    
+    # Calculate token metrics
+    input_tokens = inputs['tts_text_ids'].shape[1]  # Number of input tokens
+    output_tokens = outputs.sequences.shape[1]  # Total tokens (input + generated)
+    generated_tokens = output_tokens - input_tokens - all_prefilled_outputs['tts_lm']['last_hidden_state'].size(1)
+    
+    print(f"Prefilling text tokens: {input_tokens}")
+    print(f"Generated speech tokens: {generated_tokens}")
+    print(f"Total tokens: {output_tokens}")
+
+    # Save output (processor handles device internally)
+    txt_filename = os.path.splitext(os.path.basename(args.txt_path))[0]
+    output_path = os.path.join(args.output_dir, f"{txt_filename}_generated.wav")
+    os.makedirs(args.output_dir, exist_ok=True)
+    
+    processor.save_audio(
+        outputs.speech_outputs[0], # First (and only) batch item
+        output_path=output_path,
+    )
+    print(f"Saved output to {output_path}")
+    
+    # Print summary
+    print("\n" + "="*50)
+    print("GENERATION SUMMARY")
+    print("="*50)
+    print(f"Input file: {args.txt_path}")
+    print(f"Output file: {output_path}")
+    print(f"Speaker names: {args.speaker_name}")
+    print(f"Prefilling text tokens: {input_tokens}")
+    print(f"Generated speech tokens: {generated_tokens}")
+    print(f"Total tokens: {output_tokens}")
+    print(f"Generation time: {generation_time:.2f} seconds")
+    print(f"Audio duration: {audio_duration:.2f} seconds")
+    print(f"RTF (Real Time Factor): {rtf:.2f}x")
+    
+    print("="*50)
+
+if __name__ == "__main__":
+    main()

demo/text_examples/1p_abs.txt

@@ -0,0 +1,2 @@
+Generating long-form, multi-speaker conversational audio like podcasts poses significant challenges for traditional Text-to-Speech (TTS) systems, particularly in scalability, speaker consistency, and natural turn-taking. This report presents VibeVoice, a novel model designed to synthesize long-form speech with multiple speakers by employing the next-token diffusion framework, a unified method for modeling continuous data by autoregressively generating latent vectors via diffusion. 
+A core component of our approach is the continuous speech tokenizers operating at an ultra-low frame rate of 7.5. This tokenizer effectively preserves audio fidelity while significantly boosting computational efficiency for processing long sequences. This enables VibeVoice to synthesize long-form speech for up to 90 minutes (in a 64K context window length) with up to 4 speakers, capturing the authentic conversational "vibe" and surpassing all known open-source and closed-source dialogue models (for example, Gemini 2.5 Pro Preview TTS). Code and checkpoint are available now. 

demo/text_examples/1p_vibevoice.txt

@@ -0,0 +1 @@
+VibeVoice is a novel framework designed for generating expressive, long-form, multi-speaker conversational audio, such as podcasts, from text. It addresses significant challenges in traditional Text-to-Speech (TTS) systems, particularly in scalability, speaker consistency, and natural turn-taking. A core innovation of VibeVoice is its use of continuous speech tokenizers operating at an ultra-low frame rate of 7.5 Hz. These tokenizers efficiently preserve audio fidelity while significantly boosting computational efficiency for processing long sequences. VibeVoice employs a next-token diffusion framework, leveraging a Large Language Model to understand textual context and dialogue flow, and a diffusion head to generate high-fidelity acoustic details. The model can synthesize speech up to 90 minutes long with up to 4 distinct speakers, surpassing the typical 1-2 speaker limits of many prior models.

demo/vibevoice_realtime_colab.ipynb

@@ -0,0 +1,167 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "id": "d1785adb",
+      "metadata": {
+        "colab_type": "text",
+        "id": "view-in-github"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/microsoft/VibeVoice/blob/main/demo/vibevoice_realtime_colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "WvIaUJD2y0yU",
+      "metadata": {
+        "id": "WvIaUJD2y0yU"
+      },
+      "source": [
+        "# VibeVoice-Realtime Colab — T4 Quickstart\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "e8fTKYGx7DZk",
+      "metadata": {
+        "id": "e8fTKYGx7DZk"
+      },
+      "source": [
+        "## Step 1: Setup Environment"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "4wxJ6QHM-ZOb",
+      "metadata": {
+        "id": "4wxJ6QHM-ZOb"
+      },
+      "outputs": [],
+      "source": [
+        "# Check for T4 GPU\n",
+        "import torch\n",
+        "if torch.cuda.is_available() and \"T4\" in torch.cuda.get_device_name(0):\n",
+        "    print(\"✅ T4 GPU detected\")\n",
+        "else:\n",
+        "    print(\"\"\"\n",
+        "    ⚠️ WARNING: T4 GPU not detected\n",
+        "\n",
+        "    The recommended runtime for this Colab notebook is \"T4 GPU\".\n",
+        "\n",
+        "    To change the runtime type:\n",
+        "\n",
+        "        1. Click on \"Runtime\" in the top navigation menu\n",
+        "        2. Click on \"Change runtime type\"\n",
+        "        3. Select \"T4 GPU\"\n",
+        "        4. Click \"OK\" if a \"Disconnect and delete runtime\" window appears\n",
+        "        5. Click on \"Save\"\n",
+        "\n",
+        "    \"\"\")\n",
+        "\n",
+        "# Clone the VibeVoice repository\n",
+        "![ -d /content/VibeVoice ] || git clone --quiet --branch main --depth 1 https://github.com/microsoft/VibeVoice.git /content/VibeVoice\n",
+        "print(\"✅ Cloned VibeVoice repository\")\n",
+        "\n",
+        "# Install project dependencies\n",
+        "!uv pip --quiet install --system -e /content/VibeVoice\n",
+        "!wget -q https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -O cloudflared && chmod +x cloudflared\n",
+        "print(\"✅ Installed dependencies\")\n",
+        "\n",
+        "# Download model\n",
+        "!HF_XET_HIGH_PERFORMANCE=1 hf download microsoft/VibeVoice-Realtime-0.5B --quiet  --local-dir /content/models/VibeVoice-Realtime-0.5B > /dev/null\n",
+        "print(\"✅ Downloaded model: microsoft/VibeVoice-Realtime-0.5B\")\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "pgKlV7153Ifi",
+      "metadata": {
+        "id": "pgKlV7153Ifi"
+      },
+      "source": [
+        "## Step 2: Launch VibeVoice-Realtime Demo"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "Yc1N9EHswFxA",
+      "metadata": {
+        "id": "Yc1N9EHswFxA"
+      },
+      "outputs": [],
+      "source": [
+        "import subprocess, re, time, threading\n",
+        "\n",
+        "srv = subprocess.Popen(\n",
+        "    \"python /content/VibeVoice/demo/vibevoice_realtime_demo.py --model_path /content/models/VibeVoice-Realtime-0.5B --port 8000\",\n",
+        "    shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, universal_newlines=True,\n",
+        ")\n",
+        "cf = subprocess.Popen(\n",
+        "    \"./cloudflared tunnel --url http://localhost:8000 --no-autoupdate\",\n",
+        "    shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, universal_newlines=True,\n",
+        ")\n",
+        "\n",
+        "public_url = None\n",
+        "server_ready = False\n",
+        "url_pattern  = re.compile(r\"(https://[a-z0-9-]+\\.trycloudflare\\.com)\")\n",
+        "\n",
+        "def read_srv():\n",
+        "    global server_ready\n",
+        "    for ln in srv.stdout:\n",
+        "        print(ln.strip())\n",
+        "        if \"Uvicorn running on\" in ln:\n",
+        "            server_ready = True\n",
+        "\n",
+        "def read_cf():\n",
+        "    global public_url\n",
+        "    for ln in cf.stdout:\n",
+        "        m = url_pattern.search(ln)\n",
+        "        if m:\n",
+        "            public_url = m.group(1)\n",
+        "            break\n",
+        "\n",
+        "threading.Thread(target=read_srv, daemon=True).start()\n",
+        "threading.Thread(target=read_cf,  daemon=True).start()\n",
+        "\n",
+        "\n",
+        "while True:\n",
+        "    if server_ready and public_url:\n",
+        "        print(f\"✅ Public URL: {public_url}\\n\");\n",
+        "        public_url = None\n",
+        "    time.sleep(0.25)"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "gpuType": "T4",
+      "include_colab_link": true,
+      "machine_shape": "hm",
+      "name": "VibeVoice_Colab.ipynb",
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.10.11"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 5
+}

demo/vibevoice_realtime_demo.py

@@ -0,0 +1,17 @@
+import argparse, os, uvicorn
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument("--port", type=int, default=3000)
+    p.add_argument("--model_path", type=str, default="default_model")
+    p.add_argument("--device", type=str, default="cuda", choices=["cpu", "cuda", "mpx", "mps"])
+    p.add_argument("--reload", action="store_true", help="Reload the model or not")
+    args = p.parse_args()
+    
+    os.environ["MODEL_PATH"] = args.model_path
+    os.environ["MODEL_DEVICE"] = args.device
+
+    uvicorn.run("web.app:app", host="0.0.0.0", port=args.port, reload=args.reload)
+
+if __name__ == "__main__":
+    main()

demo/web/app.py

@@ -0,0 +1,506 @@
+import datetime
+import builtins
+import asyncio
+import json
+import os
+import threading
+import traceback
+from pathlib import Path
+from queue import Empty, Queue
+from typing import Any, Callable, Dict, Iterator, Optional, Tuple, cast
+
+import numpy as np
+import torch
+from fastapi import FastAPI, WebSocket
+from fastapi.responses import FileResponse
+from fastapi.staticfiles import StaticFiles
+from starlette.websockets import WebSocketDisconnect, WebSocketState
+
+from vibevoice.modular.modeling_vibevoice_streaming_inference import (
+    VibeVoiceStreamingForConditionalGenerationInference,
+)
+from vibevoice.processor.vibevoice_streaming_processor import (
+    VibeVoiceStreamingProcessor,
+)
+from vibevoice.modular.streamer import AudioStreamer
+
+import copy
+
+BASE = Path(__file__).parent
+SAMPLE_RATE = 24_000
+
+
+def get_timestamp():
+    timestamp = datetime.datetime.utcnow().replace(
+        tzinfo=datetime.timezone.utc
+    ).astimezone(
+        datetime.timezone(datetime.timedelta(hours=8))
+    ).strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+    return timestamp
+
+class StreamingTTSService:
+    def __init__(
+        self,
+        model_path: str,
+        device: str = "cuda",
+        inference_steps: int = 5,
+    ) -> None:
+        self.model_path = Path(model_path)
+        self.inference_steps = inference_steps
+        self.sample_rate = SAMPLE_RATE
+
+        self.processor: Optional[VibeVoiceStreamingProcessor] = None
+        self.model: Optional[VibeVoiceStreamingForConditionalGenerationInference] = None
+        self.voice_presets: Dict[str, Path] = {}
+        self.default_voice_key: Optional[str] = None
+        self._voice_cache: Dict[str, Tuple[object, Path, str]] = {}
+
+        if device == "mpx":
+            print("Note: device 'mpx' detected, treating it as 'mps'.")
+            device = "mps"        
+        if device == "mps" and not torch.backends.mps.is_available():
+            print("Warning: MPS not available. Falling back to CPU.")
+            device = "cpu"
+        self.device = device
+        self._torch_device = torch.device(device)
+
+    def load(self) -> None:
+        print(f"[startup] Loading processor from {self.model_path}")
+        self.processor = VibeVoiceStreamingProcessor.from_pretrained(str(self.model_path))
+
+        
+        # Decide dtype & attention
+        if self.device == "mps":
+            load_dtype = torch.float32
+            device_map = None
+            attn_impl_primary = "sdpa"
+        elif self.device == "cuda":
+            load_dtype = torch.bfloat16
+            device_map = 'cuda'
+            attn_impl_primary = "flash_attention_2"
+        else:
+            load_dtype = torch.float32
+            device_map = 'cpu'
+            attn_impl_primary = "sdpa"
+        print(f"Using device: {device_map}, torch_dtype: {load_dtype}, attn_implementation: {attn_impl_primary}")
+        # Load model
+        try:
+            self.model = VibeVoiceStreamingForConditionalGenerationInference.from_pretrained(
+                str(self.model_path),
+                torch_dtype=load_dtype,
+                device_map=device_map,
+                attn_implementation=attn_impl_primary,
+            )
+            
+            if self.device == "mps":
+                self.model.to("mps")
+        except Exception as e:
+            if attn_impl_primary == 'flash_attention_2':
+                print("Error loading the model. Trying to use SDPA. However, note that only flash_attention_2 has been fully tested, and using SDPA may result in lower audio quality.")
+                
+                self.model = VibeVoiceStreamingForConditionalGenerationInference.from_pretrained(
+                    str(self.model_path),
+                    torch_dtype=load_dtype,
+                    device_map=self.device,
+                    attn_implementation='sdpa',
+                )
+                print("Load model with SDPA successfully ")
+            else:
+                raise e
+
+        self.model.eval()
+
+        self.model.model.noise_scheduler = self.model.model.noise_scheduler.from_config(
+            self.model.model.noise_scheduler.config,
+            algorithm_type="sde-dpmsolver++",
+            beta_schedule="squaredcos_cap_v2",
+        )
+        self.model.set_ddpm_inference_steps(num_steps=self.inference_steps)
+
+        self.voice_presets = self._load_voice_presets()
+        preset_name = os.environ.get("VOICE_PRESET")
+        self.default_voice_key = self._determine_voice_key(preset_name)
+        self._ensure_voice_cached(self.default_voice_key)
+
+    def _load_voice_presets(self) -> Dict[str, Path]:
+        voices_dir = BASE.parent / "voices" / "streaming_model"
+        if not voices_dir.exists():
+            raise RuntimeError(f"Voices directory not found: {voices_dir}")
+
+        presets: Dict[str, Path] = {}
+        for pt_path in voices_dir.glob("*.pt"):
+            presets[pt_path.stem] = pt_path
+
+        if not presets:
+            raise RuntimeError(f"No voice preset (.pt) files found in {voices_dir}")
+
+        print(f"[startup] Found {len(presets)} voice presets")
+        return dict(sorted(presets.items()))
+
+    def _determine_voice_key(self, name: Optional[str]) -> str:
+        if name and name in self.voice_presets:
+            return name
+
+        default_key = "en-WHTest_man"
+        if default_key in self.voice_presets:
+            return default_key
+
+        first_key = next(iter(self.voice_presets))
+        print(f"[startup] Using fallback voice preset: {first_key}")
+        return first_key
+
+    def _ensure_voice_cached(self, key: str) -> Tuple[object, Path, str]:
+        if key not in self.voice_presets:
+            raise RuntimeError(f"Voice preset {key!r} not found")
+
+        if key not in self._voice_cache:
+            preset_path = self.voice_presets[key]
+            print(f"[startup] Loading voice preset {key} from {preset_path}")
+            print(f"[startup] Loading prefilled prompt from {preset_path}")
+            prefilled_outputs = torch.load(
+                preset_path,
+                map_location=self._torch_device,
+                weights_only=False,
+            )
+            self._voice_cache[key] = prefilled_outputs
+
+        return self._voice_cache[key]
+
+    def _get_voice_resources(self, requested_key: Optional[str]) -> Tuple[str, object, Path, str]:
+        key = requested_key if requested_key and requested_key in self.voice_presets else self.default_voice_key
+        if key is None:
+            key = next(iter(self.voice_presets))
+            self.default_voice_key = key
+
+        prefilled_outputs = self._ensure_voice_cached(key)
+        return key, prefilled_outputs
+
+    def _prepare_inputs(self, text: str, prefilled_outputs: object):
+        if not self.processor or not self.model:
+            raise RuntimeError("StreamingTTSService not initialized")
+
+        processor_kwargs = {
+            "text": text.strip(),
+            "cached_prompt": prefilled_outputs,
+            "padding": True,
+            "return_tensors": "pt",
+            "return_attention_mask": True,
+        }
+
+        processed = self.processor.process_input_with_cached_prompt(**processor_kwargs)
+
+        prepared = {
+            key: value.to(self._torch_device) if hasattr(value, "to") else value
+            for key, value in processed.items()
+        }
+        return prepared
+
+    def _run_generation(
+        self,
+        inputs,
+        audio_streamer: AudioStreamer,
+        errors,
+        cfg_scale: float,
+        do_sample: bool,
+        temperature: float,
+        top_p: float,
+        refresh_negative: bool,
+        prefilled_outputs,
+        stop_event: threading.Event,
+    ) -> None:
+        try:
+            self.model.generate(
+                **inputs,
+                max_new_tokens=None,
+                cfg_scale=cfg_scale,
+                tokenizer=self.processor.tokenizer,
+                generation_config={
+                    "do_sample": do_sample,
+                    "temperature": temperature if do_sample else 1.0,
+                    "top_p": top_p if do_sample else 1.0,
+                },
+                audio_streamer=audio_streamer,
+                stop_check_fn=stop_event.is_set,
+                verbose=False,
+                refresh_negative=refresh_negative,
+                all_prefilled_outputs=copy.deepcopy(prefilled_outputs),
+            )
+        except Exception as exc:  # pragma: no cover - diagnostic logging
+            errors.append(exc)
+            traceback.print_exc()
+            audio_streamer.end()
+
+    def stream(
+        self,
+        text: str,
+        cfg_scale: float = 1.5,
+        do_sample: bool = False,
+        temperature: float = 0.9,
+        top_p: float = 0.9,
+        refresh_negative: bool = True,
+        inference_steps: Optional[int] = None,
+        voice_key: Optional[str] = None,
+        log_callback: Optional[Callable[[str, Dict[str, Any]], None]] = None,
+        stop_event: Optional[threading.Event] = None,
+    ) -> Iterator[np.ndarray]:
+        if not text.strip():
+            return
+        text = text.replace("’", "'")
+        selected_voice, prefilled_outputs = self._get_voice_resources(voice_key)
+
+        def emit(event: str, **payload: Any) -> None:
+            if log_callback:
+                try:
+                    log_callback(event, **payload)
+                except Exception as exc:
+                    print(f"[log_callback] Error while emitting {event}: {exc}")
+
+        steps_to_use = self.inference_steps
+        if inference_steps is not None:
+            try:
+                parsed_steps = int(inference_steps)
+                if parsed_steps > 0:
+                    steps_to_use = parsed_steps
+            except (TypeError, ValueError):
+                pass
+        if self.model:
+            self.model.set_ddpm_inference_steps(num_steps=steps_to_use)
+        self.inference_steps = steps_to_use
+
+        inputs = self._prepare_inputs(text, prefilled_outputs)
+        audio_streamer = AudioStreamer(batch_size=1, stop_signal=None, timeout=None)
+        errors: list = []
+        stop_signal = stop_event or threading.Event()
+
+        thread = threading.Thread(
+            target=self._run_generation,
+            kwargs={
+                "inputs": inputs,
+                "audio_streamer": audio_streamer,
+                "errors": errors,
+                "cfg_scale": cfg_scale,
+                "do_sample": do_sample,
+                "temperature": temperature,
+                "top_p": top_p,
+                "refresh_negative": refresh_negative,
+                "prefilled_outputs": prefilled_outputs,
+                "stop_event": stop_signal,
+            },
+            daemon=True,
+        )
+        thread.start()
+
+        generated_samples = 0
+
+        try:
+            stream = audio_streamer.get_stream(0)
+            for audio_chunk in stream:
+                if torch.is_tensor(audio_chunk):
+                    audio_chunk = audio_chunk.detach().cpu().to(torch.float32).numpy()
+                else:
+                    audio_chunk = np.asarray(audio_chunk, dtype=np.float32)
+
+                if audio_chunk.ndim > 1:
+                    audio_chunk = audio_chunk.reshape(-1)
+
+                peak = np.max(np.abs(audio_chunk)) if audio_chunk.size else 0.0
+                if peak > 1.0:
+                    audio_chunk = audio_chunk / peak
+
+                generated_samples += int(audio_chunk.size)
+                emit(
+                    "model_progress",
+                    generated_sec=generated_samples / self.sample_rate,
+                    chunk_sec=audio_chunk.size / self.sample_rate,
+                )
+
+                chunk_to_yield = audio_chunk.astype(np.float32, copy=False)
+
+                yield chunk_to_yield
+        finally:
+            stop_signal.set()
+            audio_streamer.end()
+            thread.join()
+            if errors:
+                emit("generation_error", message=str(errors[0]))
+                raise errors[0]
+
+    def chunk_to_pcm16(self, chunk: np.ndarray) -> bytes:
+        chunk = np.clip(chunk, -1.0, 1.0)
+        pcm = (chunk * 32767.0).astype(np.int16)
+        return pcm.tobytes()
+
+
+app = FastAPI()
+
+
+@app.on_event("startup")
+async def _startup() -> None:
+    model_path = os.environ.get("MODEL_PATH")
+    if not model_path:
+        raise RuntimeError("MODEL_PATH not set in environment")
+
+    device = os.environ.get("MODEL_DEVICE", "cuda")
+    
+    service = StreamingTTSService(
+        model_path=model_path,
+        device=device
+    )
+    service.load()
+
+    app.state.tts_service = service
+    app.state.model_path = model_path
+    app.state.device = device
+    app.state.websocket_lock = asyncio.Lock()
+    print("[startup] Model ready.")
+
+
+def streaming_tts(text: str, **kwargs) -> Iterator[np.ndarray]:
+    service: StreamingTTSService = app.state.tts_service
+    yield from service.stream(text, **kwargs)
+
+@app.websocket("/stream")
+async def websocket_stream(ws: WebSocket) -> None:
+    await ws.accept()
+    text = ws.query_params.get("text", "")
+    print(f"Client connected, text={text!r}")
+    cfg_param = ws.query_params.get("cfg")
+    steps_param = ws.query_params.get("steps")
+    voice_param = ws.query_params.get("voice")
+
+    try:
+        cfg_scale = float(cfg_param) if cfg_param is not None else 1.5
+    except ValueError:
+        cfg_scale = 1.5
+    if cfg_scale <= 0:
+        cfg_scale = 1.5
+    try:
+        inference_steps = int(steps_param) if steps_param is not None else None
+        if inference_steps is not None and inference_steps <= 0:
+            inference_steps = None
+    except ValueError:
+        inference_steps = None
+
+    service: StreamingTTSService = app.state.tts_service
+    lock: asyncio.Lock = app.state.websocket_lock
+
+    if lock.locked():
+        busy_message = {
+            "type": "log",
+            "event": "backend_busy",
+            "data": {"message": "Please wait for the other requests to complete."},
+            "timestamp": get_timestamp(),
+        }
+        print("Please wait for the other requests to complete.")
+        try:
+            await ws.send_text(json.dumps(busy_message))
+        except Exception:
+            pass
+        await ws.close(code=1013, reason="Service busy")
+        return
+
+    acquired = False
+    try:
+        await lock.acquire()
+        acquired = True
+
+        log_queue: "Queue[Dict[str, Any]]" = Queue()
+
+        def enqueue_log(event: str, **data: Any) -> None:
+            log_queue.put({"event": event, "data": data})
+
+        async def flush_logs() -> None:
+            while True:
+                try:
+                    entry = log_queue.get_nowait()
+                except Empty:
+                    break
+                message = {
+                    "type": "log",
+                    "event": entry.get("event"),
+                    "data": entry.get("data", {}),
+                    "timestamp": get_timestamp(),
+                }
+                try:
+                    await ws.send_text(json.dumps(message))
+                except Exception:
+                    break
+
+        enqueue_log(
+            "backend_request_received",
+            text_length=len(text or ""),
+            cfg_scale=cfg_scale,
+            inference_steps=inference_steps,
+            voice=voice_param,
+        )
+
+        stop_signal = threading.Event()
+
+        iterator = streaming_tts(
+            text,
+            cfg_scale=cfg_scale,
+            inference_steps=inference_steps,
+            voice_key=voice_param,
+            log_callback=enqueue_log,
+            stop_event=stop_signal,
+        )
+        sentinel = object()
+        first_ws_send_logged = False
+
+        await flush_logs()
+
+        try:
+            while ws.client_state == WebSocketState.CONNECTED:
+                await flush_logs()
+                chunk = await asyncio.to_thread(next, iterator, sentinel)
+                if chunk is sentinel:
+                    break
+                chunk = cast(np.ndarray, chunk)
+                payload = service.chunk_to_pcm16(chunk)
+                await ws.send_bytes(payload)
+                if not first_ws_send_logged:
+                    first_ws_send_logged = True
+                    enqueue_log("backend_first_chunk_sent")
+                await flush_logs()
+        except WebSocketDisconnect:
+            print("Client disconnected (WebSocketDisconnect)")
+            enqueue_log("client_disconnected")
+            stop_signal.set()
+        finally:
+            stop_signal.set()
+            enqueue_log("backend_stream_complete")
+            await flush_logs()
+            try:
+                iterator_close = getattr(iterator, "close", None)
+                if callable(iterator_close):
+                    iterator_close()
+            except Exception:
+                pass
+            # clear the log queue
+            while not log_queue.empty():
+                try:
+                    log_queue.get_nowait()
+                except Empty:
+                    break
+            if ws.client_state == WebSocketState.CONNECTED:
+                await ws.close()
+            print("WS handler exit")
+    finally:
+        if acquired:
+            lock.release()
+
+
+@app.get("/")
+def index():
+    return FileResponse(BASE / "index.html")
+
+
+@app.get("/config")
+def get_config():
+    service: StreamingTTSService = app.state.tts_service
+    voices = sorted(service.voice_presets.keys())
+    return {
+        "voices": voices,
+        "default_voice": service.default_voice_key,
+    }
+