From fff0252d52f80dcb2683c0c128856210a80192fb Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 9 Dec 2025 12:45:17 +0100 Subject: [PATCH] feat: Add OpenAI-compatible TTS endpoint and engines - Implements POST /v1/audio/speech endpoint (OpenAI API compatible). - Integrates Kokoro and XTTS engines (including dependencies and implementations). - Updates main application to register new engines and router. - Adds unit tests for OpenAI compatibility. - Updates requirements.txt for new engines. --- app/config.py | 11 + app/engines/kokoro.py | 338 +++++++++++++++++++++++++++++++ app/engines/kokoro_voices.py | 205 +++++++++++++++++++ app/engines/xtts.py | 177 ++++++++++++++++ app/main.py | 26 ++- app/routers/__init__.py | 0 app/routers/openai_compatible.py | 137 +++++++++++++ requirements.txt | 20 +- tests/test_openai_api.py | 74 +++++++ 9 files changed, 985 insertions(+), 3 deletions(-) create mode 100644 app/engines/kokoro.py create mode 100644 app/engines/kokoro_voices.py create mode 100644 app/engines/xtts.py create mode 100644 app/routers/__init__.py create mode 100644 app/routers/openai_compatible.py create mode 100644 tests/test_openai_api.py diff --git a/app/config.py b/app/config.py index f9af0f4..08b578a 100644 --- a/app/config.py +++ b/app/config.py @@ -19,11 +19,22 @@ class Settings(BaseSettings): ACTIVE_ENGINES: Set[str] = {"piper"} ASSET_DIR: str = "app/asset" AUDIO_CACHE_DIR: str = "app/asset/audio" + MODELS_DIR: str = "app/models" + LOG_LEVEL: str = "INFO" # Added log level setting # Piper Engine Timeouts (in seconds) PIPER_TIMEOUT_SECONDS: int = 30 FFMPEG_TIMEOUT_SECONDS: int = 60 + # Kokoro Engine Configuration + KOKORO_DEVICE: str = "cuda" # or "cpu" + KOKORO_TIMEOUT_SECONDS: int = 30 + + # Coqui XTTS Engine Configuration + XTTS_DEVICE: str = "cuda" # or "cpu" + XTTS_ACCEPT_LICENSE: bool = False # User must opt-in + VOICES_DIR: str = "app/asset/voices" # Directory for reference speaker wavs + model_config = SettingsConfigDict(env_file=".env", env_file_encoding='utf-8') settings = Settings() diff --git a/app/engines/kokoro.py b/app/engines/kokoro.py new file mode 100644 index 0000000..29ee2d8 --- /dev/null +++ b/app/engines/kokoro.py @@ -0,0 +1,338 @@ +""" +NovaAi – TTS-Engine-Hub +engines/kokoro.py +Version: v0.1.0 + +Description: + Kokoro TTS engine adapter: 82M parameter high-quality TTS model. + Synthesizes 24kHz audio using Kokoro library, converts to OGG/MP3 via ffmpeg if needed. + Supports 54 voices across 8 languages with GPU acceleration. + +Author: Claude Code (Anthropic) +Date: 2025-12-05 +""" + +import asyncio +import subprocess +import tempfile +import os +import shutil +import logging +from typing import Optional, List +from .engine_base import TTSEngineBase +from app.config import settings +import ffmpeg + +logger = logging.getLogger(__name__) + +# Language code mapping for Kokoro +KOKORO_LANG_CODES = { + "kokoro-en-us": "a", # American English + "kokoro-en-gb": "b", # British English + "kokoro-fr": "fr", # French + "kokoro-es": "es", # Spanish + "kokoro-ja": "ja", # Japanese + "kokoro-zh": "zh", # Chinese + "kokoro-it": "it", # Italian + "kokoro-pt": "pt", # Portuguese + "kokoro-hi": "hi", # Hindi + "kokoro-ko": "ko", # Korean +} + +# Import voice metadata +from .kokoro_voices import ALL_VOICES, get_voices_for_model, get_voice_info + + +class KokoroEngine(TTSEngineBase): + def __init__(self): + self.kokoro_available = False + self.pipeline = None + self.current_lang = None + self.ffmpeg_executable = shutil.which("ffmpeg") + self.device = getattr(settings, "KOKORO_DEVICE", "cuda") + self.timeout = getattr(settings, "KOKORO_TIMEOUT_SECONDS", 30) + + # Try to import and initialize Kokoro + try: + from kokoro import KPipeline + self.KPipeline = KPipeline + self.kokoro_available = True + logger.info("Kokoro TTS library loaded successfully") + except ImportError as e: + logger.warning(f"Kokoro TTS library not available: {e}") + self.kokoro_available = False + + def _get_pipeline(self, lang_code: str): + """Get or create pipeline for specific language.""" + if not self.kokoro_available: + raise RuntimeError("Kokoro library not installed. Install with: pip install kokoro>=0.9.2") + + # Reuse pipeline if same language + if self.pipeline is not None and self.current_lang == lang_code: + return self.pipeline + + # Create new pipeline for language + try: + logger.info(f"Loading Kokoro pipeline for language code: {lang_code}") + self.pipeline = self.KPipeline(lang_code=lang_code) + self.current_lang = lang_code + return self.pipeline + except Exception as e: + logger.error(f"Failed to load Kokoro pipeline: {e}") + raise RuntimeError(f"Failed to load Kokoro pipeline for {lang_code}: {e}") + + def _run_ffmpeg_blocking(self, input_path: str, output_path: str): + """ + Wrapper for blocking ffmpeg call with error capture. + Reused from Piper engine implementation. + """ + try: + stdout, stderr = ( + ffmpeg + .input(input_path) + .output(output_path) + .run(overwrite_output=True, capture_stdout=True, capture_stderr=True) + ) + if stderr: + logger.debug(f"FFmpeg output: {stderr.decode('utf-8', errors='replace')}") + except ffmpeg.Error as e: + stderr_output = e.stderr.decode('utf-8', errors='replace') if e.stderr else "No error output" + logger.error(f"FFmpeg conversion failed: {input_path} -> {output_path}. Error: {stderr_output}") + raise RuntimeError( + f"FFmpeg conversion failed: {input_path} -> {output_path}. " + f"Error: {stderr_output}" + ) + + async def synthesize(self, text: str, speaker: str = None, model: str = None, fmt: str = "ogg") -> str: + """ + Synthesize speech from text using Kokoro TTS. + + Applies all bug fixes from Piper engine: + - Timeout protection + - Comprehensive temp file cleanup + - FFmpeg error capture + - Enhanced logging + """ + # Validation + if not self.kokoro_available: + raise RuntimeError("Kokoro library not installed. Install with: pip install kokoro>=0.9.2 soundfile") + + if not model: + model = "kokoro-en-us" # Default to American English + + if model not in KOKORO_LANG_CODES: + raise ValueError( + f"Model '{model}' not supported. Available models: {list(KOKORO_LANG_CODES.keys())}" + ) + + if not speaker: + speaker = "af_bella" # Default voice + + if speaker not in ALL_VOICES: + logger.warning( + f"Voice '{speaker}' not in known voice list. Attempting anyway. " + f"Known voices: {ALL_VOICES[:10]}..." + ) + + # Get language code + lang_code = KOKORO_LANG_CODES[model] + + # Track temp files for cleanup + temp_files_to_cleanup = [] + + try: + # Get pipeline for language + pipeline = await asyncio.to_thread(self._get_pipeline, lang_code) + + # Create WAV temp file + fd, output_wav_path = tempfile.mkstemp(suffix=".wav", prefix="kokoro_") + os.close(fd) + temp_files_to_cleanup.append(output_wav_path) + + # Log synthesis details + text_preview = text[:100] + "..." if len(text) > 100 else text + logger.debug(f"Kokoro synthesis: model={model}, voice={speaker}, text_len={len(text)}") + logger.debug(f"Text preview: {text_preview}") + + # Generate audio with timeout + try: + audio_data = await asyncio.wait_for( + asyncio.to_thread(self._synthesize_audio, pipeline, text, speaker), + timeout=self.timeout + ) + except asyncio.TimeoutError: + logger.error( + f"Kokoro synthesis timed out after {self.timeout}s. " + f"Model: {model}, Voice: {speaker}, Text length: {len(text)}" + ) + raise RuntimeError( + f"Kokoro synthesis timed out after {self.timeout}s. " + f"Text length: {len(text)} chars" + ) + + # Save audio to WAV file + import soundfile as sf + await asyncio.to_thread(sf.write, output_wav_path, audio_data, 24000) + + # Verify output created + if not os.path.exists(output_wav_path) or os.path.getsize(output_wav_path) == 0: + raise RuntimeError("Kokoro synthesis failed: output file not created or empty") + + logger.info( + f"Kokoro synthesis succeeded: {len(text)} chars -> " + f"{os.path.getsize(output_wav_path)} bytes. Model: {model}, Voice: {speaker}" + ) + + # Return WAV if requested + fmt = (fmt or "ogg").lower() + if fmt == "wav": + temp_files_to_cleanup.remove(output_wav_path) + return output_wav_path + + # FFmpeg conversion + if not self.ffmpeg_executable: + raise RuntimeError("ffmpeg not found, cannot convert audio format.") + + # Create converted file temp path + fd_conv, output_other_path = tempfile.mkstemp(suffix=f'.{fmt}', prefix="kokoro_conv_") + os.close(fd_conv) + temp_files_to_cleanup.append(output_other_path) + + logger.debug(f"Converting WAV to {fmt}: {output_wav_path} -> {output_other_path}") + + # Convert with timeout + try: + await asyncio.wait_for( + asyncio.to_thread(self._run_ffmpeg_blocking, output_wav_path, output_other_path), + timeout=60 # FFmpeg timeout + ) + except asyncio.TimeoutError: + logger.error( + f"FFmpeg conversion timed out after 60s. " + f"Input size: {os.path.getsize(output_wav_path)} bytes" + ) + raise RuntimeError( + f"FFmpeg conversion timed out after 60s. " + f"Input size: {os.path.getsize(output_wav_path)} bytes" + ) + + # Verify conversion succeeded + if not os.path.exists(output_other_path) or os.path.getsize(output_other_path) == 0: + raise RuntimeError("FFmpeg conversion failed: output file not created or empty") + + logger.info( + f"FFmpeg conversion succeeded: {os.path.getsize(output_wav_path)} bytes (WAV) -> " + f"{os.path.getsize(output_other_path)} bytes ({fmt})" + ) + + # Success! Remove converted file from cleanup (we're returning it) + temp_files_to_cleanup.remove(output_other_path) + return output_other_path + + finally: + # Cleanup all temp files + for temp_file in temp_files_to_cleanup: + try: + if os.path.exists(temp_file): + os.remove(temp_file) + logger.debug(f"Cleaned up temp file: {temp_file}") + except Exception as e: + logger.warning(f"Failed to cleanup temp file {temp_file}: {e}") + + def _synthesize_audio(self, pipeline, text: str, voice: str): + """ + Blocking synthesis function (runs in thread). + Generates audio using Kokoro pipeline. + """ + import numpy as np + + # Generate audio using pipeline + generator = pipeline(text, voice=voice) + + # Collect audio chunks + audio_chunks = [] + for gs, ps, audio in generator: + audio_chunks.append(audio) + + # Concatenate all chunks + if not audio_chunks: + raise RuntimeError("Kokoro generated no audio chunks") + + full_audio = np.concatenate(audio_chunks) + return full_audio + + def list_models(self) -> List[str]: + """Return available Kokoro language models.""" + return list(KOKORO_LANG_CODES.keys()) + + def list_voices(self, model: str = None) -> List[str]: + """Return available Kokoro voices, optionally filtered by model/language.""" + if model and model in KOKORO_LANG_CODES: + # Return voices for specific language + return sorted(get_voices_for_model(model)) + else: + # Return all voices + return sorted(ALL_VOICES) + + def healthcheck(self): + """Return health/status info for Kokoro engine.""" + status = "ok" if self.kokoro_available else "not_available" + + details = { + "status": status, + "engine": "kokoro", + "library_available": self.kokoro_available, + "device": self.device if self.kokoro_available else None, + } + + if not self.kokoro_available: + details["error"] = "Kokoro library not installed. Install with: pip install kokoro>=0.9.2 soundfile" + + return details + + async def selftest(self): + """Run self-test to verify Kokoro is working.""" + if not self.kokoro_available: + return { + "selftest": False, + "error": "Kokoro library not installed", + "engine": "kokoro" + } + + try: + # Test synthesis with default model and voice + test_text = "This is a Kokoro selftest." + audio_file = await self.synthesize( + test_text, + speaker="af_bella", + model="kokoro-en-us", + fmt="wav" + ) + + selftest_passed = os.path.exists(audio_file) and os.path.getsize(audio_file) > 0 + if selftest_passed: + os.remove(audio_file) + + return { + "selftest": selftest_passed, + "models": self.list_models(), + "voices_count": len(self.list_voices()), + "engine": "kokoro" + } + except Exception as e: + return { + "selftest": False, + "error": str(e), + "engine": "kokoro" + } + + +if __name__ == "__main__": + async def main(): + engine = KokoroEngine() + print("Healthcheck:", engine.healthcheck()) + print("Models:", engine.list_models()) + print("Voices:", engine.list_voices()[:10], "...") + print("Selftest:", await engine.selftest()) + + asyncio.run(main()) diff --git a/app/engines/kokoro_voices.py b/app/engines/kokoro_voices.py new file mode 100644 index 0000000..5cac5a9 --- /dev/null +++ b/app/engines/kokoro_voices.py @@ -0,0 +1,205 @@ +""" +NovaAi – TTS-Engine-Hub +engines/kokoro_voices.py +Version: v0.1.0 + +Description: + Voice metadata for Kokoro TTS engine. + Complete list of 54 voices across 8 languages with metadata. + +Source: https://huggingface.co/hexgrad/Kokoro-82M/blob/main/VOICES.md + +Author: Claude Code (Anthropic) +Date: 2025-12-05 +""" + +# Complete list of all 54 Kokoro voices +ALL_VOICES = [ + # American English (20 voices) + 'af_heart', 'af_alloy', 'af_aoede', 'af_bella', 'af_jessica', 'af_kore', + 'af_nicole', 'af_nova', 'af_river', 'af_sarah', 'af_sky', + 'am_adam', 'am_echo', 'am_eric', 'am_fenrir', 'am_liam', 'am_michael', + 'am_onyx', 'am_puck', 'am_santa', + + # British English (8 voices) + 'bf_alice', 'bf_emma', 'bf_isabella', 'bf_lily', + 'bm_daniel', 'bm_fable', 'bm_george', 'bm_lewis', + + # Japanese (5 voices) + 'jf_alpha', 'jf_gongitsune', 'jf_nezumi', 'jf_tebukuro', + 'jm_kumo', + + # Mandarin Chinese (8 voices) + 'zf_xiaobei', 'zf_xiaoni', 'zf_xiaoxiao', 'zf_xiaoyi', + 'zm_yunjian', 'zm_yunxi', 'zm_yunxia', 'zm_yunyang', + + # Spanish (3 voices) + 'ef_dora', 'em_alex', 'em_santa', + + # French (1 voice) + 'ff_siwis', + + # Hindi (4 voices) + 'hf_alpha', 'hf_beta', 'hm_omega', 'hm_psi', + + # Italian (2 voices) + 'if_sara', 'im_nicola', + + # Brazilian Portuguese (3 voices) + 'pf_dora', 'pm_alex', 'pm_santa', +] + +# Voice metadata with gender and language information +VOICE_METADATA = { + # American English - Female + 'af_heart': {'gender': 'F', 'language': 'en-us', 'description': 'Clear, warm female voice'}, + 'af_alloy': {'gender': 'F', 'language': 'en-us', 'description': 'Professional female voice'}, + 'af_aoede': {'gender': 'F', 'language': 'en-us', 'description': 'Expressive female voice'}, + 'af_bella': {'gender': 'F', 'language': 'en-us', 'description': 'Warm, friendly female voice'}, + 'af_jessica': {'gender': 'F', 'language': 'en-us', 'description': 'Natural female voice'}, + 'af_kore': {'gender': 'F', 'language': 'en-us', 'description': 'Energetic female voice'}, + 'af_nicole': {'gender': 'F', 'language': 'en-us', 'description': 'Smooth female voice'}, + 'af_nova': {'gender': 'F', 'language': 'en-us', 'description': 'Bright female voice'}, + 'af_river': {'gender': 'F', 'language': 'en-us', 'description': 'Calm female voice'}, + 'af_sarah': {'gender': 'F', 'language': 'en-us', 'description': 'Professional female voice'}, + 'af_sky': {'gender': 'F', 'language': 'en-us', 'description': 'Cheerful female voice'}, + + # American English - Male + 'am_adam': {'gender': 'M', 'language': 'en-us', 'description': 'Deep male voice'}, + 'am_echo': {'gender': 'M', 'language': 'en-us', 'description': 'Resonant male voice'}, + 'am_eric': {'gender': 'M', 'language': 'en-us', 'description': 'Professional male voice'}, + 'am_fenrir': {'gender': 'M', 'language': 'en-us', 'description': 'Strong male voice'}, + 'am_liam': {'gender': 'M', 'language': 'en-us', 'description': 'Friendly male voice'}, + 'am_michael': {'gender': 'M', 'language': 'en-us', 'description': 'Clear male voice'}, + 'am_onyx': {'gender': 'M', 'language': 'en-us', 'description': 'Smooth male voice'}, + 'am_puck': {'gender': 'M', 'language': 'en-us', 'description': 'Playful male voice'}, + 'am_santa': {'gender': 'M', 'language': 'en-us', 'description': 'Warm, jolly male voice'}, + + # British English - Female + 'bf_alice': {'gender': 'F', 'language': 'en-gb', 'description': 'British female voice'}, + 'bf_emma': {'gender': 'F', 'language': 'en-gb', 'description': 'British female voice'}, + 'bf_isabella': {'gender': 'F', 'language': 'en-gb', 'description': 'British female voice'}, + 'bf_lily': {'gender': 'F', 'language': 'en-gb', 'description': 'British female voice'}, + + # British English - Male + 'bm_daniel': {'gender': 'M', 'language': 'en-gb', 'description': 'British male voice'}, + 'bm_fable': {'gender': 'M', 'language': 'en-gb', 'description': 'British male voice'}, + 'bm_george': {'gender': 'M', 'language': 'en-gb', 'description': 'British male voice'}, + 'bm_lewis': {'gender': 'M', 'language': 'en-gb', 'description': 'British male voice'}, + + # Japanese - Female + 'jf_alpha': {'gender': 'F', 'language': 'ja', 'description': 'Japanese female voice'}, + 'jf_gongitsune': {'gender': 'F', 'language': 'ja', 'description': 'Japanese female voice'}, + 'jf_nezumi': {'gender': 'F', 'language': 'ja', 'description': 'Japanese female voice'}, + 'jf_tebukuro': {'gender': 'F', 'language': 'ja', 'description': 'Japanese female voice'}, + + # Japanese - Male + 'jm_kumo': {'gender': 'M', 'language': 'ja', 'description': 'Japanese male voice'}, + + # Mandarin Chinese - Female + 'zf_xiaobei': {'gender': 'F', 'language': 'zh', 'description': 'Chinese female voice'}, + 'zf_xiaoni': {'gender': 'F', 'language': 'zh', 'description': 'Chinese female voice'}, + 'zf_xiaoxiao': {'gender': 'F', 'language': 'zh', 'description': 'Chinese female voice'}, + 'zf_xiaoyi': {'gender': 'F', 'language': 'zh', 'description': 'Chinese female voice'}, + + # Mandarin Chinese - Male + 'zm_yunjian': {'gender': 'M', 'language': 'zh', 'description': 'Chinese male voice'}, + 'zm_yunxi': {'gender': 'M', 'language': 'zh', 'description': 'Chinese male voice'}, + 'zm_yunxia': {'gender': 'M', 'language': 'zh', 'description': 'Chinese male voice'}, + 'zm_yunyang': {'gender': 'M', 'language': 'zh', 'description': 'Chinese male voice'}, + + # Spanish - Female + 'ef_dora': {'gender': 'F', 'language': 'es', 'description': 'Spanish female voice'}, + + # Spanish - Male + 'em_alex': {'gender': 'M', 'language': 'es', 'description': 'Spanish male voice'}, + 'em_santa': {'gender': 'M', 'language': 'es', 'description': 'Spanish male voice'}, + + # French - Female + 'ff_siwis': {'gender': 'F', 'language': 'fr', 'description': 'French female voice'}, + + # Hindi - Female + 'hf_alpha': {'gender': 'F', 'language': 'hi', 'description': 'Hindi female voice'}, + 'hf_beta': {'gender': 'F', 'language': 'hi', 'description': 'Hindi female voice'}, + + # Hindi - Male + 'hm_omega': {'gender': 'M', 'language': 'hi', 'description': 'Hindi male voice'}, + 'hm_psi': {'gender': 'M', 'language': 'hi', 'description': 'Hindi male voice'}, + + # Italian - Female + 'if_sara': {'gender': 'F', 'language': 'it', 'description': 'Italian female voice'}, + + # Italian - Male + 'im_nicola': {'gender': 'M', 'language': 'it', 'description': 'Italian male voice'}, + + # Brazilian Portuguese - Female + 'pf_dora': {'gender': 'F', 'language': 'pt', 'description': 'Portuguese female voice'}, + + # Brazilian Portuguese - Male + 'pm_alex': {'gender': 'M', 'language': 'pt', 'description': 'Portuguese male voice'}, + 'pm_santa': {'gender': 'M', 'language': 'pt', 'description': 'Portuguese male voice'}, +} + +# Language mapping for voice filtering +VOICES_BY_LANGUAGE = { + 'en-us': [v for v in ALL_VOICES if v.startswith('a')], + 'en-gb': [v for v in ALL_VOICES if v.startswith('b')], + 'ja': [v for v in ALL_VOICES if v.startswith('j')], + 'zh': [v for v in ALL_VOICES if v.startswith('z')], + 'es': [v for v in ALL_VOICES if v.startswith('e')], + 'fr': [v for v in ALL_VOICES if v.startswith('f')], + 'hi': [v for v in ALL_VOICES if v.startswith('h')], + 'it': [v for v in ALL_VOICES if v.startswith('i')], + 'pt': [v for v in ALL_VOICES if v.startswith('p')], +} + + +def get_voices_for_model(model: str) -> list: + """ + Get voices compatible with a specific model/language. + + Args: + model: Model name (e.g., 'kokoro-en-us', 'kokoro-ja') + + Returns: + List of compatible voice IDs + """ + # Extract language code from model name + if model == 'kokoro-en-us': + return VOICES_BY_LANGUAGE['en-us'] + elif model == 'kokoro-en-gb': + return VOICES_BY_LANGUAGE['en-gb'] + elif model == 'kokoro-ja': + return VOICES_BY_LANGUAGE['ja'] + elif model == 'kokoro-zh': + return VOICES_BY_LANGUAGE['zh'] + elif model == 'kokoro-es': + return VOICES_BY_LANGUAGE['es'] + elif model == 'kokoro-fr': + return VOICES_BY_LANGUAGE['fr'] + elif model == 'kokoro-hi': + return VOICES_BY_LANGUAGE['hi'] + elif model == 'kokoro-it': + return VOICES_BY_LANGUAGE['it'] + elif model == 'kokoro-pt': + return VOICES_BY_LANGUAGE['pt'] + else: + # Return all voices if model not recognized + return ALL_VOICES + + +def get_voice_info(voice_id: str) -> dict: + """ + Get metadata for a specific voice. + + Args: + voice_id: Voice identifier (e.g., 'af_bella') + + Returns: + Dictionary with voice metadata + """ + return VOICE_METADATA.get(voice_id, { + 'gender': 'Unknown', + 'language': 'unknown', + 'description': 'No description available' + }) diff --git a/app/engines/xtts.py b/app/engines/xtts.py new file mode 100644 index 0000000..7df35a9 --- /dev/null +++ b/app/engines/xtts.py @@ -0,0 +1,177 @@ +""" +NovaAi – TTS-Engine-Hub +engines/xtts.py +Version: v0.1.0 + +Description: + Coqui XTTS v2 engine adapter. + Supports multilingual synthesis and voice cloning via reference audio. +""" + +import os +import asyncio +import logging +import torch +from .engine_base import TTSEngineBase +from app.config import settings + +logger = logging.getLogger(__name__) + +class XTTSEngine(TTSEngineBase): + def __init__(self): + logger.debug("XTTSEngine __init__ started.") + self.device = "cpu" + if torch.cuda.is_available(): + logger.debug("CUDA is available.") + if settings.XTTS_DEVICE == "cuda": + self.device = "cuda" + logger.debug(f"XTTS_DEVICE setting is 'cuda'. Using CUDA.") + else: + logger.debug(f"XTTS_DEVICE setting is '{settings.XTTS_DEVICE}'. Falling back to CPU despite CUDA availability.") + else: + logger.debug("CUDA is not available. Using CPU.") + + self.model = None + self.tts = None + + # Verify license acceptance + if not settings.XTTS_ACCEPT_LICENSE: + logger.warning("XTTS license not accepted. Engine will not load. Set XTTS_ACCEPT_LICENSE=true in .env") + return + + try: + from TTS.api import TTS + logger.debug("Coqui TTS library imported successfully.") + except ImportError: + logger.error("Coqui TTS library not found. Install 'TTS' via pip.") + return + + logger.info(f"Initializing XTTS v2 on {self.device}...") + try: + # Set environment variable to bypass TTS library's interactive license prompt + # This tells the TTS library that we agree to the terms + os.environ['COQUI_TOS_AGREED'] = '1' + + # Initialize TTS with the model name. + # This will download the model if not present. + # We use the official model name. + logger.debug(f"Calling TTS('tts_models/multilingual/multi-dataset/xtts_v2').to({self.device})...") + self.tts = TTS("tts_models/multilingual/multi-dataset/xtts_v2").to(self.device) + logger.info("XTTS v2 model loaded successfully.") + except Exception as e: + logger.error(f"Failed to load XTTS model: {e}", exc_info=True) # exc_info=True to log traceback + self.tts = None + logger.debug("XTTSEngine __init__ finished.") + + def list_models(self): + return ["xtts_v2"] + + def list_voices(self, model: str = None): + """ + Returns a list of available reference audio files (speakers) + found in the VOICES_DIR. + """ + voices_dir = settings.VOICES_DIR + if not os.path.exists(voices_dir): + return ["default"] + + # List .wav files in the voices directory + voices = [f for f in os.listdir(voices_dir) if f.lower().endswith(".wav")] + return sorted(voices) if voices else ["default"] + + def healthcheck(self): + if not settings.XTTS_ACCEPT_LICENSE: + return {"status": "license_not_accepted", "detail": "Set XTTS_ACCEPT_LICENSE=true"} + if self.tts is None: + return {"status": "error", "detail": "Model not loaded"} + return {"status": "ok", "device": self.device} + + async def synthesize(self, text: str, speaker: str = None, model: str = None, fmt: str = "wav"): + """ + Synthesize speech using XTTS v2. + + Args: + text: Text to synthesize. + speaker: Filename of the reference audio in VOICES_DIR (e.g., "my_voice.wav"). + model: Ignored (only xtts_v2 supported). + fmt: Output format (wav by default). + """ + if not self.tts: + raise RuntimeError("XTTS engine is not initialized or license not accepted.") + + # Resolve speaker/reference audio + voices_dir = settings.VOICES_DIR + if not os.path.exists(voices_dir): + os.makedirs(voices_dir, exist_ok=True) + + # precise path handling + speaker_wav = None + if speaker and speaker != "default": + potential_path = os.path.join(voices_dir, speaker) + if os.path.exists(potential_path): + speaker_wav = potential_path + else: + # Check if speaker has extension, if not try adding .wav + if not speaker.lower().endswith(".wav"): + potential_path_ext = os.path.join(voices_dir, f"{speaker}.wav") + if os.path.exists(potential_path_ext): + speaker_wav = potential_path_ext + + # Fallback if no valid speaker provided - XTTS NEEDS a speaker reference. + # We'll use a default sample if provided, or fail. + # Ideally, we should ship a default reference. + if not speaker_wav: + # Try to find *any* wav file in the dir to use as default + available = self.list_voices() + if available and available[0] != "default": + speaker_wav = os.path.join(voices_dir, available[0]) + logger.warning(f"No valid speaker '{speaker}' found. Using first available: {available[0]}") + else: + raise ValueError("XTTS requires a reference audio file (speaker). Please upload a .wav file to app/asset/voices/") + + # Output file + import tempfile + fd, output_path = tempfile.mkstemp(suffix=".wav", prefix="xtts_") + os.close(fd) + + # Run synthesis in thread pool to avoid blocking event loop + # XTTS API: tts.tts_to_file(text=..., speaker_wav=..., language=..., file_path=...) + # We need to detect language or default to English ("en") + # For now, we hardcode "en" or try to auto-detect if the library supports it, + # but tts_to_file usually requires language for multilingual models. + language = "en" # TODO: Add language parameter to API or auto-detect + + logger.info(f"Synthesizing with XTTS. Speaker: {os.path.basename(speaker_wav)}, Lang: {language}") + + try: + await asyncio.to_thread( + self.tts.tts_to_file, + text=text, + speaker_wav=speaker_wav, + language=language, + file_path=output_path + ) + except Exception as e: + logger.error(f"XTTS synthesis failed: {e}") + if os.path.exists(output_path): + os.remove(output_path) + raise RuntimeError(f"XTTS synthesis failed: {str(e)}") + + return output_path + + async def selftest(self): + try: + # Check if we have at least one reference voice + voices = self.list_voices() + if not voices or voices == ["default"]: + return {"selftest": False, "error": "No reference voices found in asset/voices", "engine": "xtts"} + + test_voice = voices[0] + output = await self.synthesize("XTTS selftest.", speaker=test_voice) + + if os.path.exists(output) and os.path.getsize(output) > 0: + os.remove(output) + return {"selftest": True, "engine": "xtts"} + return {"selftest": False, "error": "Output file empty or missing", "engine": "xtts"} + except Exception as e: + return {"selftest": False, "error": str(e), "engine": "xtts"} diff --git a/app/main.py b/app/main.py index dd3d5ca..af4b972 100644 --- a/app/main.py +++ b/app/main.py @@ -20,15 +20,28 @@ import os import base64 import shutil import uvicorn +import logging from app.config import settings from app.engines.piper import PiperEngine from app.engines.styletts import StyleTTSEngine from app.engines.chattts import ChatTTSEngine from app.engines.f5_tts import F5TTSEngine +from app.engines.kokoro import KokoroEngine +from app.engines.xtts import XTTSEngine from app.utils.text import chunk_text from app.utils.audio import concat_audio from app.utils.cache import build_cache_key +from app.routers import openai_compatible + +# Configure logging based on settings +logging.basicConfig(level=settings.LOG_LEVEL, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +# Explicitly configure uvicorn loggers +logging.getLogger("uvicorn.access").setLevel(settings.LOG_LEVEL) +logging.getLogger("uvicorn.error").setLevel(settings.LOG_LEVEL) +logging.getLogger("uvicorn.server").setLevel(settings.LOG_LEVEL) # --- Master list of all possible engine classes. --- ALL_ENGINES = { @@ -36,6 +49,8 @@ ALL_ENGINES = { "styletts": StyleTTSEngine, "chattts": ChatTTSEngine, "f5-tts": F5TTSEngine, + "kokoro": KokoroEngine, + "xtts": XTTSEngine, } def create_app(): @@ -50,14 +65,17 @@ def create_app(): app.ENGINE_REGISTRY = {} for engine_name in settings.ACTIVE_ENGINES: if engine_name in ALL_ENGINES: - print(f"Activating engine: {engine_name}") + logger.info(f"Activating engine: {engine_name}") app.ENGINE_REGISTRY[engine_name] = ALL_ENGINES[engine_name]() else: - print(f"Warning: Engine '{engine_name}' requested in config but not found in ALL_ENGINES.") + logger.warning(f"Engine '{engine_name}' requested in config but not found in ALL_ENGINES.") # Ensure the audio asset/cache directory exists. os.makedirs(settings.AUDIO_CACHE_DIR, exist_ok=True) + # Register Routers + app.include_router(openai_compatible.router) + class TTSRequest(BaseModel): text: str engine: str @@ -197,6 +215,10 @@ def create_app(): On startup, check for the existence of the models directory. This helps prevent race conditions with volume mounts. """ + if os.getenv("SKIP_MODEL_CHECK", "false").lower() == "true": + logger.info("Skipping model directory check (SKIP_MODEL_CHECK=true)") + return + model_path = "/models/piper" max_retries = 10 retry_delay = 2 # seconds diff --git a/app/routers/__init__.py b/app/routers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/routers/openai_compatible.py b/app/routers/openai_compatible.py new file mode 100644 index 0000000..2a68763 --- /dev/null +++ b/app/routers/openai_compatible.py @@ -0,0 +1,137 @@ +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import FileResponse, Response +from pydantic import BaseModel, Field +from typing import Optional, Literal +import os +import logging +from app.config import settings + +router = APIRouter() +logger = logging.getLogger(__name__) + +class OpenAISpeechRequest(BaseModel): + model: str = Field(..., description="The ID of the model to use (e.g., 'kokoro', 'tts-1')") + input: str = Field(..., description="The text to generate audio for") + voice: str = Field(..., description="The voice to use") + response_format: Optional[Literal['mp3', 'opus', 'aac', 'flac', 'wav', 'pcm']] = Field('mp3', description="The format to return audio in") + speed: Optional[float] = Field(1.0, description="The speed of the generated audio (0.25 to 4.0)") + +@router.post("/v1/audio/speech") +async def openai_speech_endpoint(req: OpenAISpeechRequest, request: Request): + """ + OpenAI-compatible speech endpoint. + Allows this API to be used as a drop-in replacement for OpenAI TTS. + """ + + # 1. Resolve Engine and Model + # Strategy: + # - If 'model' matches an active engine name exactly (e.g., 'kokoro'), use it. + # - If 'model' is 'tts-1' or 'tts-1-hd', use the first available/active engine (or a specific default if we had one). + # - If 'model' contains a separator (e.g. 'kokoro:en-us'), split it. + + engine_name = req.model.lower() + model_id = None + + # Check for engine:model format + if ":" in engine_name: + engine_name, model_id = engine_name.split(":", 1) + elif "-" in engine_name and engine_name not in request.app.ENGINE_REGISTRY: + # Try splitting by hyphen if direct match fails (e.g. kokoro-en-us -> engine: kokoro?? No, ambiguous). + # Let's stick to checking availability. + pass + + # Handle standard OpenAI model names -> Map to preferred local engine + if engine_name in ["tts-1", "tts-1-hd"]: + # Pick the first active engine as default, preferring 'kokoro' or 'xtts' if active + active_engines = list(request.app.ENGINE_REGISTRY.keys()) + if not active_engines: + raise HTTPException(status_code=503, detail="No active TTS engines available.") + + if "kokoro" in active_engines: + engine_name = "kokoro" + elif "xtts" in active_engines: + engine_name = "xtts" + else: + engine_name = active_engines[0] + + # Check engine availability + engine = request.app.ENGINE_REGISTRY.get(engine_name) + if not engine: + raise HTTPException(status_code=404, detail=f"Model/Engine '{req.model}' not found. Available: {list(request.app.ENGINE_REGISTRY.keys())}") + + # 2. Map 'voice' to 'speaker' + # Some engines are strict, others fuzzy. We pass it through. + speaker_id = req.voice + + # 3. Map 'response_format' to 'fmt' + fmt = req.response_format + if fmt == "pcm": + # We don't natively support raw PCM in all engines yet, usually wav is closest or we need ffmpeg raw + # For now, let's treat pcm as wav or raise error. + # OpenAI PCM is usually 16-bit little-endian raw. + # Let's fallback to wav for now if engine doesn't support pcm explicitly. + fmt = "wav" + + # 4. Synthesize + try: + # We rely on the engine's synthesize method. + # Note: speed is not currently supported by our BaseEngine interface. + # We are ignoring req.speed for now. + + output_path = await engine.synthesize( + text=req.input, + speaker=speaker_id, + model=model_id, # Might be None, engine uses default + fmt=fmt + ) + + if not os.path.exists(output_path): + raise RuntimeError("Synthesis finished but output file is missing.") + + except Exception as e: + logger.error(f"OpenAI API Synthesis failed: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + # 5. Return binary stream + # OpenAI returns the binary content with correct Content-Type. + + media_type_map = { + "mp3": "audio/mpeg", + "opus": "audio/opus", + "aac": "audio/aac", + "flac": "audio/flac", + "wav": "audio/wav", + "pcm": "audio/pcm" # Not standard MIME, but commonly used + } + media_type = media_type_map.get(fmt, "application/octet-stream") + + # We use FileResponse to stream the file efficiently + # We might want to add a background task to clean up the file after sending, + # but our Engine implementations often cache or handle temp files. + # The current 'synthesize' implementations in this project seem to return paths to + # temp files (Kokoro) or cached files (main.py logic). + # Since this endpoint bypasses main.py's caching logic, we might be leaking temp files + # if the engine creates unique temp files every time. + + # KokoroEngine: cleans up internal temps but returns a final temp file. It expects caller to handle it? + # Inspecting Kokoro: "temp_files_to_cleanup.remove(output_other_path) -> return output_other_path". + # So Kokoro leaves the final file for the caller. + + # We should delete the file after sending. FileResponse has a background task for this? + # No, we need to pass a background task to Starlette's Response. + + from starlette.background import BackgroundTask + + def cleanup_file(path: str): + try: + if os.path.exists(path): + os.remove(path) + logger.debug(f"Cleaned up OpenAI API temp file: {path}") + except Exception as e: + logger.warning(f"Failed to cleanup temp file {path}: {e}") + + return FileResponse( + path=output_path, + media_type=media_type, + background=BackgroundTask(cleanup_file, output_path) + ) diff --git a/requirements.txt b/requirements.txt index 724af38..e9a2a24 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,11 +10,29 @@ pydantic-settings ffmpeg-python piper-tts +# Kokoro TTS Engine +kokoro>=0.9.2 +soundfile +phonemizer +scipy +munch +# Pin compatible espeakng-loader version for misaki (kokoro dependency) +espeakng-loader>=0.2.3,<0.2.5 + +# Coqui XTTS Engine +TTS +# Pin transformers to version compatible with TTS library +transformers<4.42.0 f5-tts -torch +# Pin torch to <2.6 to avoid weights_only loading issues with TTS library +torch<2.6 torchaudio +# Numba/Numpy compatibility for XTTS/Torch +numba<0.58 +numpy<1.25 + # Development & Testing pytest-cov pytest-asyncio diff --git a/tests/test_openai_api.py b/tests/test_openai_api.py new file mode 100644 index 0000000..1da693d --- /dev/null +++ b/tests/test_openai_api.py @@ -0,0 +1,74 @@ +import pytest +from fastapi.testclient import TestClient +from unittest.mock import AsyncMock, MagicMock +from app.routers.openai_compatible import OpenAISpeechRequest + +def test_openai_speech_endpoint_success(app_client, app_instance, tmp_path): + # 1. Setup Mock Engine + mock_engine = MagicMock() + # Mock synthesis to return a dummy file path + dummy_file = tmp_path / "test_output.wav" + dummy_file.write_bytes(b"fake audio data") + mock_engine.synthesize = AsyncMock(return_value=str(dummy_file)) + + # Inject mock engine into registry + app_instance.ENGINE_REGISTRY["mock-engine"] = mock_engine + + # 2. Make Request + payload = { + "model": "mock-engine", + "input": "Hello OpenAI", + "voice": "default", + "response_format": "wav" + } + response = app_client.post("/v1/audio/speech", json=payload) + + # 3. Assertions + assert response.status_code == 200 + assert response.headers["content-type"] == "audio/wav" + assert response.content == b"fake audio data" + + # Verify engine call + mock_engine.synthesize.assert_called_once_with( + text="Hello OpenAI", + speaker="default", + model=None, + fmt="wav" + ) + +def test_openai_speech_endpoint_engine_not_found(app_client): + payload = { + "model": "non-existent-engine", + "input": "Test", + "voice": "default" + } + response = app_client.post("/v1/audio/speech", json=payload) + assert response.status_code == 404 + assert "not found" in response.json()["detail"] + +def test_openai_speech_endpoint_tts_1_mapping(app_client, app_instance, tmp_path): + # Setup Mock Engine for 'kokoro' (simulating it's active) + mock_engine = MagicMock() + dummy_file = tmp_path / "tts1_output.mp3" + dummy_file.write_bytes(b"mp3 data") + mock_engine.synthesize = AsyncMock(return_value=str(dummy_file)) + + # Inject into registry + app_instance.ENGINE_REGISTRY["kokoro"] = mock_engine + + # Request 'tts-1' -> should map to 'kokoro' + payload = { + "model": "tts-1", + "input": "Mapping test", + "voice": "alloy", + "response_format": "mp3" + } + response = app_client.post("/v1/audio/speech", json=payload) + + assert response.status_code == 200 + assert response.headers["content-type"] == "audio/mpeg" + + mock_engine.synthesize.assert_called_once() + # Check that speaker passed through + args, kwargs = mock_engine.synthesize.call_args + assert kwargs["speaker"] == "alloy"