277 lines
9.2 KiB
Python
277 lines
9.2 KiB
Python
"""
|
||
Text-to-Speech engines for MARK XL.
|
||
|
||
EdgeTTS – free Microsoft TTS (internet required, no API key)
|
||
Kokoro – fully offline neural TTS (~330 MB model)
|
||
ElevenLabs – cloud API (API key required, best quality)
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import threading
|
||
from typing import Callable, Optional
|
||
|
||
import numpy as np
|
||
import sounddevice as sd
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Audio playback helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _to_numpy(samples) -> np.ndarray:
|
||
"""Convert samples to float32 numpy array.
|
||
|
||
Handles both numpy arrays and PyTorch tensors (Kokoro >= 0.9).
|
||
|
||
PyTorch built against numpy 1.x raises RuntimeError('Numpy is not available')
|
||
when numpy 2.x is installed. The .tolist() fallback always works regardless
|
||
of PyTorch / numpy version pairing.
|
||
"""
|
||
if hasattr(samples, "detach"): # PyTorch tensor
|
||
t = samples.detach().cpu().float()
|
||
try:
|
||
return t.numpy() # fast path (compatible versions)
|
||
except RuntimeError:
|
||
# PyTorch/numpy version mismatch — convert via Python list (always safe)
|
||
return np.asarray(t.tolist(), dtype=np.float32)
|
||
return np.asarray(samples, dtype=np.float32)
|
||
|
||
|
||
def _compress_silence(
|
||
arr: np.ndarray,
|
||
sample_rate: int = 24_000,
|
||
max_silence_ms: int = 300, # cap long pauses to this many ms
|
||
threshold: float = 0.0015, # RMS below this = silence (very conservative)
|
||
) -> np.ndarray:
|
||
"""
|
||
Shorten excessively long silent segments in Kokoro audio.
|
||
|
||
Kokoro can produce 1-2 s pauses after punctuation. This caps them
|
||
to max_silence_ms while leaving natural short pauses intact.
|
||
|
||
threshold is intentionally low (0.0015) so soft consonants like
|
||
's', 'f', 'h' are never misidentified as silence.
|
||
"""
|
||
max_samp = int(max_silence_ms * sample_rate / 1000)
|
||
frame_len = 240 # ~10 ms at 24 kHz
|
||
out: list[np.ndarray] = []
|
||
silent_acc = 0
|
||
|
||
for i in range(0, len(arr), frame_len):
|
||
chunk = arr[i : i + frame_len]
|
||
if np.sqrt(np.mean(chunk ** 2) + 1e-12) < threshold:
|
||
silent_acc += len(chunk)
|
||
if silent_acc <= max_samp:
|
||
out.append(chunk)
|
||
else:
|
||
silent_acc = 0
|
||
out.append(chunk)
|
||
|
||
return np.concatenate(out) if out else arr
|
||
|
||
|
||
def _play_np(samples, sample_rate: int) -> None:
|
||
"""Play float32 mono (or stereo) audio via sounddevice.
|
||
Accepts numpy arrays or PyTorch tensors.
|
||
"""
|
||
sd.play(_to_numpy(samples), sample_rate)
|
||
sd.wait()
|
||
|
||
|
||
def _play_audio_bytes(audio_bytes: bytes) -> None:
|
||
"""Decode MP3/WAV/OGG bytes and play via sounddevice (uses miniaudio)."""
|
||
import miniaudio
|
||
decoded = miniaudio.decode(
|
||
audio_bytes,
|
||
output_format=miniaudio.SampleFormat.FLOAT32,
|
||
nchannels=1,
|
||
)
|
||
samples = np.array(decoded.samples, dtype=np.float32)
|
||
sd.play(samples, decoded.sample_rate)
|
||
sd.wait()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Engines
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class EdgeTTSEngine:
|
||
"""Microsoft EdgeTTS – free, requires internet."""
|
||
|
||
def __init__(self, voice: str = "en-US-GuyNeural"):
|
||
self.voice = voice
|
||
|
||
def speak(self, text: str) -> None:
|
||
loop = asyncio.new_event_loop()
|
||
try:
|
||
audio_bytes = loop.run_until_complete(self._synth(text))
|
||
finally:
|
||
loop.close()
|
||
if audio_bytes:
|
||
_play_audio_bytes(audio_bytes)
|
||
|
||
async def _synth(self, text: str) -> bytes:
|
||
import edge_tts
|
||
comm = edge_tts.Communicate(text, self.voice)
|
||
buf = bytearray()
|
||
async for chunk in comm.stream():
|
||
if chunk["type"] == "audio":
|
||
buf.extend(chunk["data"])
|
||
return bytes(buf)
|
||
|
||
|
||
# Kokoro voice prefix → KPipeline lang_code mapping
|
||
_KOKORO_LANG_CODES = {
|
||
"a": "a", # American English (af_*, am_*)
|
||
"b": "b", # British English (bf_*, bm_*)
|
||
"j": "j", # Japanese (jf_*, jm_*)
|
||
"z": "z", # Mandarin Chinese (zf_*, zm_*)
|
||
"s": "s", # Spanish (sf_*, sm_*)
|
||
"f": "f", # French (ff_*, fm_*)
|
||
"h": "h", # Hindi (hf_*, hm_*)
|
||
"i": "i", # Italian (if_*, im_*)
|
||
"p": "p", # Brazilian Portuguese
|
||
"r": "r", # Russian (rf_*, rm_*)
|
||
"e": "e", # German (ef_*, em_*)
|
||
}
|
||
|
||
|
||
class KokoroTTSEngine:
|
||
"""Fully offline Kokoro neural TTS.
|
||
|
||
Model (~330 MB) is downloaded from HuggingFace on first use.
|
||
Subsequent runs load from local cache instantly.
|
||
"""
|
||
|
||
def __init__(self, voice: str = "af_heart", speed: float = 1.2):
|
||
self.voice = voice
|
||
self.speed = speed # 1.0 = normal, 1.2 = 20% faster (recommended)
|
||
self._pipeline = None
|
||
self._lock = threading.Lock()
|
||
# Eager init: download/load model now so it's ready before first speak()
|
||
self._init()
|
||
|
||
# Detect lang_code from voice prefix (e.g. "af_heart" → "a")
|
||
@property
|
||
def _lang_code(self) -> str:
|
||
prefix = self.voice[0].lower() if self.voice else "a"
|
||
return _KOKORO_LANG_CODES.get(prefix, "a")
|
||
|
||
def _init(self) -> None:
|
||
if self._pipeline is not None:
|
||
return
|
||
try:
|
||
from kokoro import KPipeline
|
||
lang = self._lang_code
|
||
print(f"[TTS] Loading Kokoro pipeline (lang='{lang}', voice='{self.voice}')…")
|
||
print("[TTS] Kokoro model will be downloaded (~330 MB) if not cached.")
|
||
self._pipeline = KPipeline(lang_code=lang)
|
||
print("[TTS] Kokoro pipeline ready.")
|
||
except Exception as e:
|
||
print(f"[TTS] Kokoro init error: {e}")
|
||
raise
|
||
|
||
def speak(self, text: str) -> None:
|
||
with self._lock:
|
||
if self._pipeline is None:
|
||
self._init()
|
||
for _, _, audio in self._pipeline(text, voice=self.voice, speed=self.speed):
|
||
if audio is not None:
|
||
arr = _to_numpy(audio)
|
||
arr = _compress_silence(arr) # shorten long punctuation pauses
|
||
if arr.size > 0:
|
||
_play_np(arr, 24000)
|
||
|
||
|
||
class ElevenLabsTTSEngine:
|
||
"""ElevenLabs cloud TTS – API key required."""
|
||
|
||
def __init__(self, api_key: str, voice_id: str = "pNInz6obpgDQGcFmaJgB"):
|
||
self.api_key = api_key
|
||
self.voice_id = voice_id
|
||
|
||
def speak(self, text: str) -> None:
|
||
import requests
|
||
headers = {
|
||
"xi-api-key": self.api_key,
|
||
"Content-Type": "application/json",
|
||
}
|
||
payload = {
|
||
"text": text,
|
||
"model_id": "eleven_multilingual_v2",
|
||
"voice_settings": {"stability": 0.5, "similarity_boost": 0.75},
|
||
}
|
||
resp = requests.post(
|
||
f"https://api.elevenlabs.io/v1/text-to-speech/{self.voice_id}",
|
||
json=payload, headers=headers, timeout=30,
|
||
)
|
||
resp.raise_for_status()
|
||
_play_audio_bytes(resp.content)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Thread-safe player wrapper
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class TTSPlayer:
|
||
"""
|
||
Wraps any *Engine. Exposes a blocking speak() method
|
||
meant to be called from a dedicated background thread.
|
||
"""
|
||
|
||
def __init__(self, engine):
|
||
self._engine = engine
|
||
self._playing = False
|
||
self._lock = threading.Lock()
|
||
|
||
@property
|
||
def is_playing(self) -> bool:
|
||
return self._playing
|
||
|
||
def speak(
|
||
self,
|
||
text: str,
|
||
on_start: Optional[Callable] = None,
|
||
on_done: Optional[Callable] = None,
|
||
) -> None:
|
||
"""Synthesise and play text. BLOCKING – call from a dedicated thread."""
|
||
try:
|
||
with self._lock:
|
||
self._playing = True
|
||
if on_start:
|
||
on_start()
|
||
self._engine.speak(text)
|
||
except Exception as e:
|
||
print(f"[TTS] Error: {e}")
|
||
finally:
|
||
with self._lock:
|
||
self._playing = False
|
||
if on_done:
|
||
on_done()
|
||
|
||
def stop(self) -> None:
|
||
sd.stop()
|
||
with self._lock:
|
||
self._playing = False
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Factory
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def create_tts_player(config: dict) -> TTSPlayer:
|
||
engine_name = config.get("tts_engine", "edgetts").lower()
|
||
if engine_name == "kokoro":
|
||
voice = config.get("tts_voice", "af_heart")
|
||
speed = float(config.get("tts_speed", 1.2))
|
||
engine = KokoroTTSEngine(voice=voice, speed=speed)
|
||
elif engine_name == "elevenlabs":
|
||
api_key = config.get("elevenlabs_api_key", "")
|
||
voice_id = config.get("tts_voice", "pNInz6obpgDQGcFmaJgB")
|
||
engine = ElevenLabsTTSEngine(api_key=api_key, voice_id=voice_id)
|
||
else: # edgetts (default)
|
||
voice = config.get("tts_voice", "en-US-GuyNeural")
|
||
engine = EdgeTTSEngine(voice=voice)
|
||
return TTSPlayer(engine)
|