Fix XTTS loader compatibility and add default voice

worker/Dockerfile

@@ -1,7 +1,8 @@
 FROM nvidia/cuda:12.1.0-runtime-ubuntu22.04 AS base
+ENV PYTHONUNBUFFERED=1
 WORKDIR /app
 COPY requirements.worker.txt .
-RUN apt-get update && apt-get install -y python3-pip ffmpeg
+RUN apt-get update && apt-get install -y python3-pip ffmpeg espeak-ng
 RUN pip3 install -r requirements.worker.txt
 COPY . .
 CMD ["python3","main.py"]

worker/engine/audio_export.py

@@ -1,19 +1,39 @@
 from pydub import AudioSegment
 import io
+import numpy as np
 
-def convert_audio(raw_bytes: bytes, fmt: str):
-    # raw mono 32-bit float fake waveform
+def convert_audio(audio_data, fmt: str, sample_rate: int = 24000):
+    """
+    Converts raw audio data (numpy array or list of floats) to the target format.
+    Assumes mono audio.
+    """
+    # Ensure numpy array
+    if not isinstance(audio_data, np.ndarray):
+        audio_data = np.array(audio_data)
+
+    # Check if float and normalize/convert to int16
+    if audio_data.dtype.kind == 'f':
+        # Clip to Avoid wrap-around
+        audio_data = np.clip(audio_data, -1.0, 1.0)
+        # Convert to 16-bit PCM
+        audio_data = (audio_data * 32767).astype(np.int16)
+    
     seg = AudioSegment(
-        raw_bytes,
-        frame_rate=22050,
-        sample_width=4,
+        audio_data.tobytes(),
+        frame_rate=sample_rate,
+        sample_width=2, # 16-bit
         channels=1
     )
-    buf=io.BytesIO()
+    
+    buf = io.BytesIO()
     seg.export(buf, format=fmt)
-    mime={
-        "wav":"audio/wav",
-        "mp3":"audio/mpeg",
-        "ogg":"audio/ogg"
-    }.get(fmt,"audio/wav")
-    return buf.getvalue(), mime
+    
+    mime = {
+        "wav": "audio/wav",
+        "mp3": "audio/mpeg",
+        "ogg": "audio/ogg",
+        "flac": "audio/flac",
+        "aac": "audio/aac"
+    }.get(fmt, "audio/wav")
+    
+    return buf.getvalue(), mime

worker/engine/xtts2_loader.py

@@ -1,10 +1,163 @@
-# Dummy XTTS2 logic placeholder
-# Replace with real TTS model loading
+import os
+# Auto-agree to Coqui TOS (Must be before imports)
+os.environ["COQUI_TOS_AGREED"] = "1"
+
+import json
+import base64
+import tempfile
+import requests
+import torch
+import numpy as np
+
+# Singleton for lazy loading
+_model = None
+
+def _allow_xtts_config_pickle():
+    """Allow loading XTTS configs with torch >=2.6 safe loading."""
+    add_safe = getattr(torch.serialization, "add_safe_globals", None)
+    if not add_safe:
+        return
+    allowed = []
+    try:
+        from TTS.tts.configs.xtts_config import XttsConfig
+        from TTS.tts.models.xtts import XttsAudioConfig
+        allowed += [XttsConfig, XttsAudioConfig]
+    except Exception as e:
+        print(f"⚠️ Could not register safe globals for XTTS config: {e}")
+    try:
+        import TTS.config.shared_configs as shared_configs
+        allowed += [v for v in shared_configs.__dict__.values() if isinstance(v, type)]
+    except Exception as e:
+        print(f"⚠️ Could not register shared config globals: {e}")
+    try:
+        import TTS.tts.models.xtts as xtts_models
+        allowed += [v for v in xtts_models.__dict__.values() if isinstance(v, type)]
+    except Exception as e:
+        print(f"⚠️ Could not register XTTS model globals: {e}")
+    if allowed:
+        add_safe(allowed)
+
+def get_model():
+    global _model
+    if _model is None:
+        print("⏳ Loading XTTS Model (Lazy Load)....")
+        # Lazy Import to prevent startup hang
+        from TTS.api import TTS
+        _allow_xtts_config_pickle()
+        
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        print(f"🔧 XTTS Running on: {device}")
+        
+        # Load Model (download if needed)
+        # Using default XTTS v2 model
+        _model = TTS("tts_models/multilingual/multi-dataset/xtts_v2").to(device)
+        print("✅ XTTS Model loaded successfully.")
+    return _model
 
 def synthesize(job: dict):
-    # return artificial sine wave placeholder
-    import numpy as np
-    sr=22050
-    t=np.linspace(0,0.3,int(sr*0.3))
-    tone=(0.1*np.sin(2*np.pi*440*t)).astype('float32')
-    return tone.tobytes()
+    from langdetect import detect
+    model = get_model()
+    ...
+    
+    text = job.get("input")
+    if not text:
+        raise ValueError("No input text provided")
+        
+    # Language handling
+    language = job.get("language")
+    if not language:
+        try:
+            # Simple detection
+            detected = detect(text)
+            # XTTS expects 2-letter codes usually. 
+            # We assume detected is valid or mapped if needed.
+            # Supported: en, es, fr, de, it, pt, pl, tr, ru, nl, cs, ar, zh-cn, ja, hu, ko
+            language = detected
+            print(f"🌍 Auto-detected language: {language}")
+        except:
+            language = "en"
+            print("⚠️ Language detection failed, using 'en'")
+            
+    # Speaker Handling
+    speaker_wav = None
+    temp_files = []
+    
+    try:
+        # Priority 1: Direct URL
+        if job.get("voice_sample_url"):
+            try:
+                print(f"⬇️ Downloading voice sample from {job['voice_sample_url']}")
+                r = requests.get(job["voice_sample_url"], timeout=10)
+                r.raise_for_status()
+                t = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
+                t.write(r.content)
+                t.close()
+                speaker_wav = t.name
+                temp_files.append(t.name)
+            except Exception as e:
+                print(f"❌ Failed to download voice sample: {e}")
+
+        # Priority 2: Base64
+        if not speaker_wav and job.get("voice_sample_base64"):
+            try:
+                b64 = job["voice_sample_base64"]
+                decoded = base64.b64decode(b64)
+                t = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
+                t.write(decoded)
+                t.close()
+                speaker_wav = t.name
+                temp_files.append(t.name)
+            except Exception as e:
+                print(f"❌ Failed to decode base64 voice: {e}")
+
+        # Priority 3: Registry / Local File
+        if not speaker_wav:
+            voice_id = job.get("voice", "auto")
+            if voice_id and voice_id != "auto":
+                # Look in worker/voices/
+                base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+                voice_path = os.path.join(base_dir, "voices", f"{voice_id}.wav")
+                
+                # Check for other extensions if wav missing
+                if not os.path.exists(voice_path):
+                    for ext in [".mp3", ".ogg", ".m4a"]:
+                        p = os.path.join(base_dir, "voices", f"{voice_id}{ext}")
+                        if os.path.exists(p):
+                            voice_path = p
+                            break
+                            
+                if os.path.exists(voice_path):
+                    speaker_wav = voice_path
+                    print(f"🗣️ Using registered voice: {voice_id}")
+                else:
+                    print(f"⚠️ Voice '{voice_id}' not found in registry.")
+
+        # Priority 4: Default/Auto Voice
+        if not speaker_wav:
+            # Fallback to a default file if it exists
+            base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+            default_path = os.path.join(base_dir, "voices", "default.wav")
+            if os.path.exists(default_path):
+                speaker_wav = default_path
+                print("⚠️ Using default.wav")
+            else:
+                # If completely nothing, we can't synthesize with XTTS
+                # Unless we use speaker_idxs (only for multi-speaker models w/o cloning?)
+                # XTTS v2 IS zero-shot, needs reference.
+                raise ValueError("No speaker reference found (url, base64, registry, or default.wav)")
+
+        # Run Inference
+        print(f"🎤 Synthesizing: '{text[:30]}...' Lang: {language}")
+        
+        # XTTS API returns List[float]
+        wav = model.tts(text=text, speaker_wav=speaker_wav, language=language)
+        
+        return wav
+
+    finally:
+        # Cleanup
+        for f in temp_files:
+            try:
+                os.remove(f)
+            except:
+                pass

worker/main.py

@@ -1,9 +1,14 @@
+print("DEBUG: Starting worker...", flush=True)
 import time, json, os
+print("DEBUG: Imported stdlib", flush=True)
 from core.queue_worker import fetch_job, store_result
+print("DEBUG: Imported queue_worker", flush=True)
 from engine.xtts2_loader import synthesize
+print("DEBUG: Imported xtts2_loader", flush=True)
 from engine.audio_export import convert_audio
+print("DEBUG: Imported audio_export", flush=True)
 
-print("Worker gestartet. Warte auf Jobs…")
+print("Worker gestartet. Warte auf Jobs…", flush=True)
 
 while True:
     job = fetch_job()
@@ -11,6 +16,13 @@ while True:
         time.sleep(0.1)
         continue
 
-    audio = synthesize(job)
-    out, mime = convert_audio(audio, job.get("format","wav"))
-    store_result(job["job_id"], out, mime)
+    try:
+        start = time.time()
+        print(f"🔄 Processing Job {job['job_id']}...", flush=True)
+        audio = synthesize(job)
+        out, mime = convert_audio(audio, job.get("format","wav"))
+        store_result(job["job_id"], out, mime)
+        print(f"✅ Job {job['job_id']} done in {time.time()-start:.2f}s")
+    except Exception as e:
+        print(f"❌ Error processing job {job.get('job_id')}: {e}")
+        # Optional: Store error state if protocol supports it

worker/requirements.worker.txt

@@ -3,3 +3,8 @@ redis
 torch
 numpy
 requests
+transformers==4.42.4
+TTS==0.22.0
+scipy
+langdetect
+torchcodec

worker/voices/README.md

@@ -0,0 +1,17 @@
+# Voice Registry
+
+Place `.wav` files here to register them as permanent voices.
+
+## Usage
+
+If you place a file named `narrator.wav` in this directory:
+
+1.  Restart the worker (or mount this volume dynamically).
+2.  Send a request with `"voice": "narrator"`.
+
+The system will use this file as the speaker reference for XTTS cloning.
+
+## Formats
+
+Supported formats: `.wav`, `.mp3`, `.ogg`, `.m4a`.
+Recommended: Mono, 22050Hz or 24000Hz WAV (16-bit).